From 942cee1023a4ebf9e62cea0243eda924172f5438 Mon Sep 17 00:00:00 2001 From: cristi8 Date: Wed, 7 Jul 2021 13:27:35 +0300 Subject: [PATCH 01/21] webhooks server component: db migration, service, tests --- packages/server/modules/webhooks/index.js | 0 .../migrations/20210701180000-webhooks.js | 36 ++++++ .../modules/webhooks/services/webhooks.js | 66 +++++++++++ .../modules/webhooks/tests/webhooks.spec.js | 105 ++++++++++++++++++ 4 files changed, 207 insertions(+) create mode 100644 packages/server/modules/webhooks/index.js create mode 100644 packages/server/modules/webhooks/migrations/20210701180000-webhooks.js create mode 100644 packages/server/modules/webhooks/services/webhooks.js create mode 100644 packages/server/modules/webhooks/tests/webhooks.spec.js diff --git a/packages/server/modules/webhooks/index.js b/packages/server/modules/webhooks/index.js new file mode 100644 index 000000000..e69de29bb diff --git a/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js b/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js new file mode 100644 index 000000000..5cccf8bf0 --- /dev/null +++ b/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js @@ -0,0 +1,36 @@ +/* istanbul ignore file */ +'use strict' + +exports.up = async knex => { + await knex.schema.createTable( 'webhooks_config', table => { + table.string( 'id' ).primary( ) + table.string( 'streamId', 10 ).references( 'id' ).inTable( 'streams' ).onDelete( 'cascade' ) + table.string( 'url' ) + table.string( 'description' ) + table.jsonb( 'events' ) + table.string( 'secret' ) + table.boolean( 'enabled' ).defaultTo( true ) + + table.index( 'streamId' ) + } ) + + await knex.schema.createTable( 'webhooks_events', table => { + table.string( 'id' ).primary( ) + table.string( 'webhookId' ).references( 'id' ).inTable( 'webhooks_config' ).onDelete( 'cascade' ) + + table.integer( 'status' ).notNullable( ).defaultTo( 0 ) + table.string( 'statusInfo' ).notNullable( ).defaultTo( 'Pending' ) + + table.integer( 'retryCount' ).notNullable( ).defaultTo( 0 ) + table.timestamp( 'lastUpdate' ).notNullable( ).defaultTo( knex.fn.now( ) ) + + table.string( 'payload' ) + + table.index( 'webhookId' ) + } ) +} + +exports.down = async knex => { + await knex.schema.dropTableIfExists( 'webhooks_events' ) + await knex.schema.dropTableIfExists( 'webhooks_config' ) +} diff --git a/packages/server/modules/webhooks/services/webhooks.js b/packages/server/modules/webhooks/services/webhooks.js new file mode 100644 index 000000000..4f8d82323 --- /dev/null +++ b/packages/server/modules/webhooks/services/webhooks.js @@ -0,0 +1,66 @@ +'use strict' + +const appRoot = require( 'app-root-path' ) +const knex = require( `${appRoot}/db/knex` ) +const crs = require( 'crypto-random-string' ) + +const WebhooksConfig = ( ) => knex( 'webhooks_config' ) +const WebhooksEvents = ( ) => knex( 'webhooks_events' ) + +module.exports = { + + async createWebhook( { streamId, url, description, secret, enabled, events } ) { + // TODO: limit max number of webhooks for a stream to 100 (github has a 20 limit per event) + + let [ id ] = await WebhooksConfig( ).returning( 'id' ).insert( { + id: crs( { length: 10 } ), + streamId, + url, + description, + secret, + enabled, + events: events + } ) + return id + }, + + async getWebhook( { id } ) { + // TODO: get webhook object + summary of event history (last event status, etc) + return await WebhooksConfig().select( '*' ).where( { id } ).first() + }, + + async updateWebhook( { id, url, description, secret, enabled, events } ) { + let fieldsToUpdate = {} + if ( url !== undefined ) fieldsToUpdate.url = url + if ( description !== undefined ) fieldsToUpdate.description = description + if ( secret !== undefined ) fieldsToUpdate.secret = secret + if ( enabled !== undefined ) fieldsToUpdate.enabled = enabled + if ( events !== undefined ) fieldsToUpdate.events = events + + let [ res ] = await WebhooksConfig( ) + .returning( 'id' ) + .where( { id } ) + .update( fieldsToUpdate ) + return res + }, + + async deleteWebhook( { id } ) { + return await WebhooksConfig( ).where( { id } ).del( ) + }, + + async getStreamWebhooks( { streamId } ) { + // TODO: also get summary of event history for each webhook (last event status, etc) + return await WebhooksConfig( ).select( '*' ).where( { streamId } ) + }, + + async dispatchStreamEvent( { streamId, event, eventPayload } ) { + // TODO: get all enabled webhooks that are registered to this event, enrich the payload with stream info and create rows in WebhooksEvents for sending + }, + + async getLastWebhookEvents( { webhookId, limit } ) { + if ( !limit ) { + limit = 100 + } + return await WebhooksEvents( ).select( '*' ).where( { webhookId } ).orderBy( 'lastUpdate', 'desc' ).limit( limit ) + }, +} diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js new file mode 100644 index 000000000..3445fa1ad --- /dev/null +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -0,0 +1,105 @@ +/* istanbul ignore file */ +const chai = require( 'chai' ) +const chaiHttp = require( 'chai-http' ) +const assert = require( 'assert' ) + +const appRoot = require( 'app-root-path' ) +const { init } = require( `${appRoot}/app` ) +const knex = require( `${appRoot}/db/knex` ) + +const expect = chai.expect +chai.use( chaiHttp ) + + +const { createWebhook, getStreamWebhooks, getLastWebhookEvents, getWebhook, updateWebhook, deleteWebhook, dispatchStreamEvent } = require( '../services/webhooks' ) +const { createUser } = require( '../../core/services/users' ) +const { createStream, getStream } = require( '../../core/services/streams' ) + +describe( 'Webhooks', ( ) => { + let userOne = { + name: 'User', + email: 'user@gmail.com', + password: 'jdsadjsadasfdsa' + } + + let streamOne = { + name: 'stream', + description: 'stream', + isPublic: true + } + + let webhookOne = { + streamId: null, // filled in `before` + url: 'http://localhost:42/non-existent', + description: 'test wh', + secret: 'secret', + enabled: true, + events: { + 'commit_create': true, + 'commit_update': true + } + } + + before( async ( ) => { + await knex.migrate.rollback( ) + await knex.migrate.latest( ) + await init() + + userOne.id = await createUser( userOne ) + streamOne.ownerId = userOne.id + streamOne.id = await createStream( streamOne ) + webhookOne.streamId = streamOne.id + } ) + + after( async ( ) => { + // await knex.migrate.rollback( ) + } ) + + describe( 'Create, Read, Update, Delete Webhooks', ( ) => { + it( 'Should create a webhook', async ( ) => { + webhookOne.id = await createWebhook( webhookOne ) + expect( webhookOne ).to.have.property( 'id' ) + expect( webhookOne.id ).to.not.be.null + } ) + + it( 'Should get a webhook', async ( ) => { + let webhook = await getWebhook( { id: webhookOne.id } ) + expect( webhook ).to.not.be.null + expect( webhook ).to.have.property( 'url' ) + expect( webhook.url ).to.equal( webhookOne.url ) + } ) + + it( 'Should update a webhook', async ( ) => { + let newUrl = 'http://localhost:42/new-url' + await updateWebhook( { id: webhookOne.id, url: newUrl } ) + let webhook = await getWebhook( { id: webhookOne.id } ) + expect( webhook ).to.not.be.null + expect( webhook ).to.have.property( 'url' ) + expect( webhook.url ).to.equal( newUrl ) + } ) + + it( 'Should delete a webhook', async ( ) => { + await deleteWebhook( { id: webhookOne.id } ) + let webhook = await getWebhook( { id: webhookOne.id } ) + expect( webhook ).to.be.undefined + } ) + + it( 'Should get webooks for stream', async ( ) => { + let streamWebhooks = await getStreamWebhooks( { streamId: streamOne.id } ) + expect( streamWebhooks ).to.have.lengthOf( 0 ) + + webhookOne.id = await createWebhook( webhookOne ) + streamWebhooks = await getStreamWebhooks( { streamId: streamOne.id } ) + expect( streamWebhooks ).to.have.lengthOf( 1 ) + expect( streamWebhooks[0] ).to.have.property( 'url' ) + expect( streamWebhooks[0].url ).to.equal( webhookOne.url ) + } ) + + //it( 'Should dispatch and get events', async () => { + // await dispatchStreamEvent( { streamId: streamOne.id, event: 'commit_create', eventPayload: 'payload123' } ) + // let lastEvents = getLastWebhookEvents( { webhookId: webhookOne.id } ) + // expect( lastEvents ).to.have.lengthOf( 1 ) + // expect( lastEvents[0].payload ).to.equal( 'payload123' ) + //} ) + } ) +} ) From 8577d6a7a983f9e26c5dd67dcb08d0e7bd156997 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Fri, 9 Jul 2021 15:09:49 +0100 Subject: [PATCH 02/21] feat(webhooks): initial gql queries & mutations --- .../webhooks/graph/resolvers/webhooks.js | 34 ++++++++++ .../webhooks/graph/schemas/webhooks.graphql | 62 +++++++++++++++++++ .../modules/webhooks/services/webhooks.js | 5 +- 3 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 packages/server/modules/webhooks/graph/resolvers/webhooks.js create mode 100644 packages/server/modules/webhooks/graph/schemas/webhooks.graphql diff --git a/packages/server/modules/webhooks/graph/resolvers/webhooks.js b/packages/server/modules/webhooks/graph/resolvers/webhooks.js new file mode 100644 index 000000000..95735b402 --- /dev/null +++ b/packages/server/modules/webhooks/graph/resolvers/webhooks.js @@ -0,0 +1,34 @@ +const { createWebhook, getWebhook, updateWebhook, deleteWebhook, getStreamWebhooks, getLastWebhookEvents } = require( '../../services/webhooks' ) + + +module.exports = { + Stream: { + async webhooks( parent, args, context, info ) { + if ( args.id ) { + let wh = await getWebhook( { id: args.id } ) + let items = wh ? [ wh ] : [] + return { items, totalCount: items.length } + } + + let items = await getStreamWebhooks( { streamId: parent.id } ) + return { items, totalCount: items.length } + } + }, + + Mutation: { + async webhookCreate( parent, args, context, info ) { + let id = await createWebhook( { streamId: args.webhook.streamId, url: args.webhook.url, description: args.webhook.description, secret: args.webhook.secret, enabled: args.webhook.enabled !== false, events: args.webhook.events } ) + + return id + }, + async webhookUpdate( parent, args, context, info ) { + let updated = await updateWebhook( { id: args.webhook.id, url: args.webhook.url, description: args.webhook.description, secret: args.webhook.secret, enabled: args.webhook.enabled !== false, events: args.webhook.events } ) + + return !!updated + }, + async webhookDelete( parent, args, context, info ) { + let deleted = await deleteWebhook( { id: args.id } ) + return !!deleted + } + } +} diff --git a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql new file mode 100644 index 000000000..8b63d57f7 --- /dev/null +++ b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql @@ -0,0 +1,62 @@ +extend type Stream { + webhooks(id: String): WebhookCollection + @hasRole(role: "stream:owner") + @hasScope(scope: "streams:write") +} + +extend type Mutation { + """ + Creates a new webhook on a stream + """ + webhookCreate(webhook: WebhookCreateInput!): String! + @hasRole(role: "stream:owner") + @hasScope(scope: "streams:write") + + """ + Updates an existing webhook + """ + webhookUpdate(webhook: WebhookUpdateInput!): String! + @hasRole(role: "stream:owner") + @hasScope(scope: "streams:write") + + """ + Deletes an existing webhook + """ + webhookDelete(id: String!): String! + @hasRole(role: "stream:owner") + @hasScope(scope: "streams:write") +} + +type WebhookCollection { + totalCount: Int + cursor: String + items: [Webhook] +} + +type Webhook { + id: String! + streamId: String! + url: String! + description: String + events: JSONObject! + secret: String + enabled: Boolean +} + +input WebhookCreateInput { + streamId: String! + url: String! + description: String + events: JSONObject! + secret: String + enabled: Boolean +} + +input WebhookUpdateInput { + id: String! + url: String + description: String + secret: String + enabled: Boolean + events: JSONObject +} diff --git a/packages/server/modules/webhooks/services/webhooks.js b/packages/server/modules/webhooks/services/webhooks.js index 4f8d82323..5f9eb7d65 100644 --- a/packages/server/modules/webhooks/services/webhooks.js +++ b/packages/server/modules/webhooks/services/webhooks.js @@ -11,7 +11,7 @@ module.exports = { async createWebhook( { streamId, url, description, secret, enabled, events } ) { // TODO: limit max number of webhooks for a stream to 100 (github has a 20 limit per event) - + let [ id ] = await WebhooksConfig( ).returning( 'id' ).insert( { id: crs( { length: 10 } ), streamId, @@ -23,7 +23,7 @@ module.exports = { } ) return id }, - + async getWebhook( { id } ) { // TODO: get webhook object + summary of event history (last event status, etc) return await WebhooksConfig().select( '*' ).where( { id } ).first() @@ -61,6 +61,7 @@ module.exports = { if ( !limit ) { limit = 100 } + return await WebhooksEvents( ).select( '*' ).where( { webhookId } ).orderBy( 'lastUpdate', 'desc' ).limit( limit ) }, } From 0141385adf3fd67a180eb8d97d94d683e4d68150 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Fri, 9 Jul 2021 15:35:08 +0100 Subject: [PATCH 03/21] feat(webhooks): remove `secret` from query --- packages/server/modules/webhooks/graph/resolvers/webhooks.js | 1 + packages/server/modules/webhooks/graph/schemas/webhooks.graphql | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/modules/webhooks/graph/resolvers/webhooks.js b/packages/server/modules/webhooks/graph/resolvers/webhooks.js index 95735b402..9cf03151d 100644 --- a/packages/server/modules/webhooks/graph/resolvers/webhooks.js +++ b/packages/server/modules/webhooks/graph/resolvers/webhooks.js @@ -28,6 +28,7 @@ module.exports = { }, async webhookDelete( parent, args, context, info ) { let deleted = await deleteWebhook( { id: args.id } ) + return !!deleted } } diff --git a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql index 8b63d57f7..8e902a832 100644 --- a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql +++ b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql @@ -39,7 +39,6 @@ type Webhook { url: String! description: String events: JSONObject! - secret: String enabled: Boolean } From 8ebc87419a65c3fe51458877c2c1f6c42d1d5d14 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Fri, 9 Jul 2021 16:06:24 +0100 Subject: [PATCH 04/21] feat(webhooks): add event history to gql query --- .../webhooks/graph/resolvers/webhooks.js | 11 ++++++++++- .../webhooks/graph/schemas/webhooks.graphql | 17 +++++++++++++++++ .../modules/webhooks/services/webhooks.js | 6 ++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/server/modules/webhooks/graph/resolvers/webhooks.js b/packages/server/modules/webhooks/graph/resolvers/webhooks.js index 9cf03151d..c70dd9c20 100644 --- a/packages/server/modules/webhooks/graph/resolvers/webhooks.js +++ b/packages/server/modules/webhooks/graph/resolvers/webhooks.js @@ -1,4 +1,4 @@ -const { createWebhook, getWebhook, updateWebhook, deleteWebhook, getStreamWebhooks, getLastWebhookEvents } = require( '../../services/webhooks' ) +const { createWebhook, getWebhook, updateWebhook, deleteWebhook, getStreamWebhooks, getLastWebhookEvents, getWebhookEventsCount } = require( '../../services/webhooks' ) module.exports = { @@ -15,6 +15,15 @@ module.exports = { } }, + Webhook: { + async history( parent, args, context, info ) { + let items = await getLastWebhookEvents( { webhookId: parent.id, limit: args.limit } ) + let totalCount = await getWebhookEventsCount( { webhookId: parent.id } ) + + return { items, totalCount } + } + }, + Mutation: { async webhookCreate( parent, args, context, info ) { let id = await createWebhook( { streamId: args.webhook.streamId, url: args.webhook.url, description: args.webhook.description, secret: args.webhook.secret, enabled: args.webhook.enabled !== false, events: args.webhook.events } ) diff --git a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql index 8e902a832..22d013fca 100644 --- a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql +++ b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql @@ -40,6 +40,7 @@ type Webhook { description: String events: JSONObject! enabled: Boolean + history(limit: Int! = 25): WebhookEventCollection } input WebhookCreateInput { @@ -59,3 +60,19 @@ input WebhookUpdateInput { enabled: Boolean events: JSONObject } + +type WebhookEventCollection{ + totalCount: Int + cursor: String + items: [WebhookEvent] +} + +type WebhookEvent { + id: String! + webhookId: String! + status: Int! + statusInfo: String! + retryCount: Int! + lastUpdate: DateTime! + payload: String! +} diff --git a/packages/server/modules/webhooks/services/webhooks.js b/packages/server/modules/webhooks/services/webhooks.js index 5f9eb7d65..3c847492c 100644 --- a/packages/server/modules/webhooks/services/webhooks.js +++ b/packages/server/modules/webhooks/services/webhooks.js @@ -64,4 +64,10 @@ module.exports = { return await WebhooksEvents( ).select( '*' ).where( { webhookId } ).orderBy( 'lastUpdate', 'desc' ).limit( limit ) }, + + async getWebhookEventsCount( { webhookId } ) { + let [ res ] = await WebhooksEvents().count().where( { webhookId } ) + + return parseInt( res.count ) + } } From ddc86eb1578b70f1fe2e4c01765966fbd45e6001 Mon Sep 17 00:00:00 2001 From: cristi8 Date: Mon, 12 Jul 2021 11:32:54 +0300 Subject: [PATCH 05/21] Webhooks - microservice + hooks into activityStreams --- .../modules/activitystream/services/index.js | 8 + .../migrations/20210701180000-webhooks.js | 2 +- .../modules/webhooks/services/webhooks.js | 14 +- .../modules/webhooks/tests/webhooks.spec.js | 12 +- packages/webhook-service/.eslintrc.json | 41 +++ packages/webhook-service/Dockerfile | 21 ++ packages/webhook-service/package-lock.json | 272 ++++++++++++++++++ packages/webhook-service/package.json | 22 ++ packages/webhook-service/src/knex.js | 8 + packages/webhook-service/src/main.js | 110 +++++++ packages/webhook-service/src/webhookCaller.js | 49 ++++ 11 files changed, 551 insertions(+), 8 deletions(-) create mode 100644 packages/webhook-service/.eslintrc.json create mode 100644 packages/webhook-service/Dockerfile create mode 100644 packages/webhook-service/package-lock.json create mode 100644 packages/webhook-service/package.json create mode 100644 packages/webhook-service/src/knex.js create mode 100644 packages/webhook-service/src/main.js create mode 100644 packages/webhook-service/src/webhookCaller.js diff --git a/packages/server/modules/activitystream/services/index.js b/packages/server/modules/activitystream/services/index.js index c02ac4a94..908eb5fa0 100644 --- a/packages/server/modules/activitystream/services/index.js +++ b/packages/server/modules/activitystream/services/index.js @@ -3,6 +3,7 @@ const appRoot = require( 'app-root-path' ) const knex = require( `${appRoot}/db/knex` ) +const { dispatchStreamEvent } = require( '../../webhooks/services/webhooks' ) const StreamActivity = () => knex( 'stream_activity' ) const StreamAcl = ( ) => knex( 'stream_acl' ) @@ -19,6 +20,13 @@ module.exports = { message } await StreamActivity( ).insert( dbObject ) + if ( streamId ) { + let webhooksPayload = { + 'event_name': actionType, + 'data': info + } + dispatchStreamEvent( { streamId, event: actionType, eventPayload: webhooksPayload } ) + } }, async getStreamActivity( { streamId, actionType, after, before, limit } ) { diff --git a/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js b/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js index 5cccf8bf0..fcbd82a3e 100644 --- a/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js +++ b/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js @@ -21,12 +21,12 @@ exports.up = async knex => { table.integer( 'status' ).notNullable( ).defaultTo( 0 ) table.string( 'statusInfo' ).notNullable( ).defaultTo( 'Pending' ) - table.integer( 'retryCount' ).notNullable( ).defaultTo( 0 ) table.timestamp( 'lastUpdate' ).notNullable( ).defaultTo( knex.fn.now( ) ) table.string( 'payload' ) table.index( 'webhookId' ) + table.index( 'status' ) } ) } diff --git a/packages/server/modules/webhooks/services/webhooks.js b/packages/server/modules/webhooks/services/webhooks.js index 4f8d82323..51f06c2b6 100644 --- a/packages/server/modules/webhooks/services/webhooks.js +++ b/packages/server/modules/webhooks/services/webhooks.js @@ -54,7 +54,19 @@ module.exports = { }, async dispatchStreamEvent( { streamId, event, eventPayload } ) { - // TODO: get all enabled webhooks that are registered to this event, enrich the payload with stream info and create rows in WebhooksEvents for sending + let { rows } = await knex.raw( ` + SELECT * FROM webhooks_config WHERE "streamId" = ? + `, [ streamId ] ) + for ( let wh of rows ) { + if ( !( event in wh.events ) ) + continue + + await WebhooksEvents( ).insert( { + id: crs( { length: 20 } ), + webhookId: wh.id, + payload: eventPayload + } ) + } }, async getLastWebhookEvents( { webhookId, limit } ) { diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js index 3445fa1ad..fa32d46ce 100644 --- a/packages/server/modules/webhooks/tests/webhooks.spec.js +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -95,11 +95,11 @@ describe( 'Webhooks', ( ) => { expect( streamWebhooks[0].url ).to.equal( webhookOne.url ) } ) - //it( 'Should dispatch and get events', async () => { - // await dispatchStreamEvent( { streamId: streamOne.id, event: 'commit_create', eventPayload: 'payload123' } ) - // let lastEvents = getLastWebhookEvents( { webhookId: webhookOne.id } ) - // expect( lastEvents ).to.have.lengthOf( 1 ) - // expect( lastEvents[0].payload ).to.equal( 'payload123' ) - //} ) + it( 'Should dispatch and get events', async () => { + await dispatchStreamEvent( { streamId: streamOne.id, event: 'commit_create', eventPayload: 'payload123' } ) + let lastEvents = await getLastWebhookEvents( { webhookId: webhookOne.id } ) + expect( lastEvents ).to.have.lengthOf( 1 ) + expect( lastEvents[0].payload ).to.equal( 'payload123' ) + } ) } ) } ) diff --git a/packages/webhook-service/.eslintrc.json b/packages/webhook-service/.eslintrc.json new file mode 100644 index 000000000..55da7fb5a --- /dev/null +++ b/packages/webhook-service/.eslintrc.json @@ -0,0 +1,41 @@ +{ + "env": { + "browser": true, + "commonjs": true, + "es2020": true + }, + "parserOptions": { + "ecmaVersion": 11 + }, + "ignorePatterns": ["node_modules/*"], + "rules": { + "arrow-spacing": [ + 2, + { + "before": true, + "after": true + } + ], + "array-bracket-spacing": [2, "always"], + "object-curly-spacing": [1, "always"], + "block-spacing": [2, "always"], + "camelcase": [ + 1, + { + "properties": "always" + } + ], + "space-in-parens": [2, "always"], + "keyword-spacing": 2, + "semi": [1, "never"], + "quotes": [1, "single"], + "indent": ["error", 2], + "space-unary-ops": [ + 2, + { + "words": true, + "nonwords": false + } + ] + } +} diff --git a/packages/webhook-service/Dockerfile b/packages/webhook-service/Dockerfile new file mode 100644 index 000000000..5728ab266 --- /dev/null +++ b/packages/webhook-service/Dockerfile @@ -0,0 +1,21 @@ +FROM node:14.16.0-buster-slim as node + +RUN apt-get update && apt-get install -y \ + tini \ + && rm -rf /var/lib/apt/lists/* + +ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.8.0/wait /wait +RUN chmod +x /wait + +ARG NODE_ENV=production +ENV NODE_ENV=${NODE_ENV} + +WORKDIR /app + +COPY packages/webhook-service/package*.json ./ +RUN npm ci + +COPY packages/webhook-service/src . + +ENTRYPOINT [ "tini", "--" ] +CMD ["node", "main.js"] diff --git a/packages/webhook-service/package-lock.json b/packages/webhook-service/package-lock.json new file mode 100644 index 000000000..784eccda1 --- /dev/null +++ b/packages/webhook-service/package-lock.json @@ -0,0 +1,272 @@ +{ + "name": "@speckle/webhook-service", + "version": "2.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, + "colorette": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-1.2.1.tgz", + "integrity": "sha512-puCDz0CzydiSYOrnXpz/PKd69zRrribezjtE9yd4zvytoRc8+RY/KJPvtPFKZS3E3wP6neGyMe0vOTlHO5L3Pw==" + }, + "commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==" + }, + "debug": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", + "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "requires": { + "ms": "2.1.2" + } + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" + }, + "esm": { + "version": "3.2.25", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz", + "integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==" + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "getopts": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/getopts/-/getopts-2.2.5.tgz", + "integrity": "sha512-9jb7AW5p3in+IiJWhQiZmmwkpLaR/ccTWdWQCtZM66HJcHHLegowh4q4tSD7gouUyeNvFWRavfK9GXosQHDpFA==" + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "interpret": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", + "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==" + }, + "is-core-module": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.4.0.tgz", + "integrity": "sha512-6A2fkfq1rfeQZjxrZJGerpLCTHRNEBiSgnu0+obeJpEPZRUooHgsizvzv0ZjJwOz3iWIHdJtVWJ/tmPr3D21/A==", + "requires": { + "has": "^1.0.3" + } + }, + "knex": { + "version": "0.95.7", + "resolved": "https://registry.npmjs.org/knex/-/knex-0.95.7.tgz", + "integrity": "sha512-J2X79td0NAcreTyWVmmHHretz5Ox705FHywddjkT3esTtmggphjcfDoaXym18xtsLdjzOvEb53WB/58lqcF14w==", + "requires": { + "colorette": "1.2.1", + "commander": "^7.1.0", + "debug": "4.3.2", + "escalade": "^3.1.1", + "esm": "^3.2.25", + "getopts": "2.2.5", + "interpret": "^2.2.0", + "lodash": "^4.17.21", + "pg-connection-string": "2.5.0", + "rechoir": "^0.7.0", + "resolve-from": "^5.0.0", + "tarn": "^3.0.1", + "tildify": "2.0.0" + } + }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" + }, + "pg": { + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.6.0.tgz", + "integrity": "sha512-qNS9u61lqljTDFvmk/N66EeGq3n6Ujzj0FFyNMGQr6XuEv4tgNTXvJQTfJdcvGit5p5/DWPu+wj920hAJFI+QQ==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.5.0", + "pg-pool": "^3.3.0", + "pg-protocol": "^1.5.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + } + }, + "pg-connection-string": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", + "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-pool": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.3.0.tgz", + "integrity": "sha512-0O5huCql8/D6PIRFAlmccjphLYWC+JIzvUhSzXSpGaf+tjTZc4nn+Lr7mLXBbFJfvwbP0ywDv73EiaBsxn7zdg==" + }, + "pg-protocol": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.5.0.tgz", + "integrity": "sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.4.tgz", + "integrity": "sha512-YmuA56alyBq7M59vxVBfPJrGSozru8QAdoNlWuW3cz8l+UX3cWge0vTvjKhsSHSJpo3Bom8/Mm6hf0TR5GY0+w==", + "requires": { + "split2": "^3.1.1" + } + }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" + }, + "postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "rechoir": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.7.0.tgz", + "integrity": "sha512-ADsDEH2bvbjltXEP+hTIAmeFekTFK0V2BTxMkok6qILyAJEXV0AFfoWcAq4yfll5VdIMd/RVXq0lR+wQi5ZU3Q==", + "requires": { + "resolve": "^1.9.0" + } + }, + "resolve": { + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.20.0.tgz", + "integrity": "sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A==", + "requires": { + "is-core-module": "^2.2.0", + "path-parse": "^1.0.6" + } + }, + "resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==" + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, + "split2": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/split2/-/split2-3.2.2.tgz", + "integrity": "sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==", + "requires": { + "readable-stream": "^3.0.0" + } + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "requires": { + "safe-buffer": "~5.2.0" + } + }, + "tarn": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/tarn/-/tarn-3.0.1.tgz", + "integrity": "sha512-6usSlV9KyHsspvwu2duKH+FMUhqJnAh6J5J/4MITl8s94iSUQTLkJggdiewKv4RyARQccnigV48Z+khiuVZDJw==" + }, + "tildify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", + "integrity": "sha512-Cc+OraorugtXNfs50hU9KS369rFXCfgGLpfCfvlc+Ud5u6VWmUQsOAa9HbTvheQdYnrdJqqv1e5oIqXppMYnSw==" + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + } + } +} diff --git a/packages/webhook-service/package.json b/packages/webhook-service/package.json new file mode 100644 index 000000000..1289b2bcc --- /dev/null +++ b/packages/webhook-service/package.json @@ -0,0 +1,22 @@ +{ + "name": "@speckle/webhook-service", + "version": "2.0.0", + "description": "Component to handle calling external webhooks", + "main": "index.js", + "repository": { + "type": "git", + "url": "git+https://github.com/specklesystems/speckle-server.git" + }, + "bugs": { + "url": "https://github.com/specklesystems/speckle-server/issues" + }, + "homepage": "https://github.com/specklesystems/speckle-server#readme", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "dependencies": { + "knex": "^0.95.7", + "node-fetch": "^2.6.1", + "pg": "^8.6.0" + } +} diff --git a/packages/webhook-service/src/knex.js b/packages/webhook-service/src/knex.js new file mode 100644 index 000000000..ff2e57488 --- /dev/null +++ b/packages/webhook-service/src/knex.js @@ -0,0 +1,8 @@ +'use strict' + +module.exports = require( 'knex' )( { + client: 'pg', + connection: process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@localhost/speckle', + pool: { min: 1, max: 1 } + // migrations are in managed in the server package +} ) diff --git a/packages/webhook-service/src/main.js b/packages/webhook-service/src/main.js new file mode 100644 index 000000000..f059e1ac5 --- /dev/null +++ b/packages/webhook-service/src/main.js @@ -0,0 +1,110 @@ +'use strict' + +const crypto = require( 'crypto' ) +const knex = require( './knex' ) + +const { makeNetworkRequest } = require( './webhookCaller' ) + +async function startTask() { + let { rows } = await knex.raw( ` + UPDATE webhooks_events + SET + "status" = 1, + "lastUpdate" = NOW() + FROM ( + SELECT "id" FROM webhooks_events + WHERE "status" = 0 + ORDER BY "lastUpdate" ASC + LIMIT 1 + ) as task + WHERE webhooks_events."id" = task."id" + RETURNING webhooks_events."id" + ` ) + return rows[0] +} + +async function doTask( task ) { + try { + let { rows } = await knex.raw( ` + SELECT + ev.payload as evt, + cnf.id as wh_id, cnf.url as wh_url, cnf.description as wh_desc, cnf.secret as wh_secret, cnf.enabled as wh_enabled, + stm.id as stm_id, stm.name as stm_name, stm.description as stm_desc, stm."isPublic" as stm_pub + FROM webhooks_events ev + INNER JOIN webhooks_config cnf ON ev."webhookId" = cnf.id + INNER JOIN streams stm ON cnf."streamId" = stm.id + WHERE ev.id = ? + LIMIT 1 + `, [ task.id ] ) + let info = rows[0] + if ( !info ) { + throw new Error( 'Internal error: DB inconsistent' ) + } + if ( !info.wh_enabled ) return + + let fullPayload = { + stream: { id: info.stm_id, name: info.stm_name, description: info.stm_desc, isPublic: info.stm_pub }, + webhook: { id: info.wh_id, url: info.wh_url, description: info.wh_desc }, + event: JSON.parse( info.evt ) + } + + let postData = { payload: JSON.stringify( fullPayload ) } + + let signature = crypto.createHmac( 'sha256', info.wh_secret || '' ).update( postData.payload ).digest( 'hex' ) + let postHeaders = { 'X-WEBHOOK-SIGNATURE': signature } + + console.log( `Callin webhook ${fullPayload.stream.id} : ${fullPayload.event.event_name} at ${fullPayload.webhook.url}...` ) + let result = await makeNetworkRequest( { url: info.wh_url, data: postData, headersData: postHeaders } ) + + console.log( ` Result: ${JSON.stringify( result )}` ) + + if ( !result.success ) { + throw new Error( result.error ) + } + + await knex.raw( ` + UPDATE webhooks_events + SET + "status" = 2, + "lastUpdate" = NOW(), + "statusInfo" = 'Webhook called' + WHERE "id" = ? + `, [ task.id ] ) + } catch ( err ) { + await knex.raw( ` + UPDATE webhooks_events + SET + "status" = 3, + "lastUpdate" = NOW(), + "statusInfo" = ? + WHERE "id" = ? + `, [ err.toString(), task.id ] ) + } + +} + +async function tick() { + try { + let task = await startTask() + if ( !task ) { + setTimeout( tick, 1000 ) + return + } + + await doTask( task ) + + // Check for another task very soon + setTimeout( tick, 10 ) + } catch ( err ) { + console.log( 'Error executing task: ', err ) + setTimeout( tick, 5000 ) + } +} + + +async function main() { + console.log( 'Starting Webhook Service...' ) + tick() +} + +main() diff --git a/packages/webhook-service/src/webhookCaller.js b/packages/webhook-service/src/webhookCaller.js new file mode 100644 index 000000000..057d9b4e5 --- /dev/null +++ b/packages/webhook-service/src/webhookCaller.js @@ -0,0 +1,49 @@ +'use strict' + +// Ignore invalid/self-signed https certificate errors for the entire process +process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0' + +const fetch = require( 'node-fetch' ) +var debug = require( 'debug' )( 'speckle' ) + +async function makeNetworkRequest( { url, data, headersData } ) { + let httpSuccessCodes = [ 200 ] + let headers = { 'Content-Type': 'application/json' } + for ( let k in headersData ) headers[ k ] = headersData[ k ] + + debug( 'POST request to:', url ) + let t0 = Date.now() + + try { + let response = await fetch( url, { + method: 'POST', + body: JSON.stringify( data ), + headers: headers, + follow: 2, // follow max 2 redirects (fetch defaults to 20) + timeout: 10 * 1000, // timeout after 10sec (defauls to no timeout) + size: 500 * 1000, // 500kb max response size, to accomodate various error responses (defaults to no limit) + } ).then( async res => ( { status: res.status, body: await res.text() } ) ) + + //console.log( 'Server response:', response ) + let error = httpSuccessCodes.indexOf( response.status ) === -1 ? `HTTP response code: ${response.status}` : '' + let success = httpSuccessCodes.indexOf( response.status ) !== -1 + return { + success: success, + error: error, + duration: ( Date.now() - t0 ) / 1000, + responseCode: response.status, + responseBody: response.body + } + } catch ( e ) { + return { + success: false, + error: e.toString(), + duration: ( Date.now() - t0 ) / 1000, + responseCode: null, + responseBody: null + } + } +} + +module.exports = { makeNetworkRequest } + From e7554b0c29eeb5d4808a0e8da4ae84e9fab53efa Mon Sep 17 00:00:00 2001 From: cristi8 Date: Mon, 12 Jul 2021 11:46:18 +0300 Subject: [PATCH 06/21] docker-compose update to include webhook-service --- docker-compose-speckle.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker-compose-speckle.yml b/docker-compose-speckle.yml index 8fb1b565d..fc242fb4d 100644 --- a/docker-compose-speckle.yml +++ b/docker-compose-speckle.yml @@ -39,3 +39,12 @@ services: environment: DEBUG: "preview-service:*" PG_CONNECTION_STRING: "postgres://speckle:speckle@postgres/speckle" + + webhook-service: + build: + context: . + dockerfile: packages/webhook-service/Dockerfile + restart: always + environment: + DEBUG: "webhook-service:*" + PG_CONNECTION_STRING: "postgres://speckle:speckle@postgres/speckle" From 46565605fd53772cc06db8758623bcb8eed2e7c4 Mon Sep 17 00:00:00 2001 From: cristi8 Date: Tue, 13 Jul 2021 10:25:00 +0300 Subject: [PATCH 07/21] webhook events should be presented as a list of strings --- .../webhooks/graph/schemas/webhooks.graphql | 6 ++-- .../modules/webhooks/services/webhooks.js | 35 +++++++++++++++---- .../modules/webhooks/tests/webhooks.spec.js | 32 ++++++++++++++--- packages/webhook-service/package.json | 3 +- 4 files changed, 61 insertions(+), 15 deletions(-) diff --git a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql index 22d013fca..163fb8522 100644 --- a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql +++ b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql @@ -38,7 +38,7 @@ type Webhook { streamId: String! url: String! description: String - events: JSONObject! + events: [String]! enabled: Boolean history(limit: Int! = 25): WebhookEventCollection } @@ -47,7 +47,7 @@ input WebhookCreateInput { streamId: String! url: String! description: String - events: JSONObject! + events: [String]! secret: String enabled: Boolean } @@ -58,7 +58,7 @@ input WebhookUpdateInput { description: String secret: String enabled: Boolean - events: JSONObject + events: [String] } type WebhookEventCollection{ diff --git a/packages/server/modules/webhooks/services/webhooks.js b/packages/server/modules/webhooks/services/webhooks.js index 14b79e633..a6c3c0807 100644 --- a/packages/server/modules/webhooks/services/webhooks.js +++ b/packages/server/modules/webhooks/services/webhooks.js @@ -7,10 +7,17 @@ const crs = require( 'crypto-random-string' ) const WebhooksConfig = ( ) => knex( 'webhooks_config' ) const WebhooksEvents = ( ) => knex( 'webhooks_events' ) +let MAX_STREAM_WEBHOOKS = 100 + module.exports = { async createWebhook( { streamId, url, description, secret, enabled, events } ) { - // TODO: limit max number of webhooks for a stream to 100 (github has a 20 limit per event) + let streamWebhookCount = await module.exports.getStreamWebhooksCount( { streamId } ) + if ( streamWebhookCount >= MAX_STREAM_WEBHOOKS ) { + throw new Error( `Maximum number of webhooks for a stream reached (${MAX_STREAM_WEBHOOKS})` ) + } + + let eventsObj = Object.assign( {}, ...events.map( ( x ) => ( { [x]: true } ) ) ) let [ id ] = await WebhooksConfig( ).returning( 'id' ).insert( { id: crs( { length: 10 } ), @@ -19,14 +26,17 @@ module.exports = { description, secret, enabled, - events: events + events: eventsObj } ) return id }, async getWebhook( { id } ) { - // TODO: get webhook object + summary of event history (last event status, etc) - return await WebhooksConfig().select( '*' ).where( { id } ).first() + let webhook = await WebhooksConfig().select( '*' ).where( { id } ).first() + if ( webhook ) { + webhook.events = Object.keys( webhook.events ) + } + return webhook }, async updateWebhook( { id, url, description, secret, enabled, events } ) { @@ -35,7 +45,10 @@ module.exports = { if ( description !== undefined ) fieldsToUpdate.description = description if ( secret !== undefined ) fieldsToUpdate.secret = secret if ( enabled !== undefined ) fieldsToUpdate.enabled = enabled - if ( events !== undefined ) fieldsToUpdate.events = events + if ( events !== undefined ) { + let eventsObj = Object.assign( {}, ...events.map( ( x ) => ( { [x]: true } ) ) ) + fieldsToUpdate.events = eventsObj + } let [ res ] = await WebhooksConfig( ) .returning( 'id' ) @@ -49,8 +62,16 @@ module.exports = { }, async getStreamWebhooks( { streamId } ) { - // TODO: also get summary of event history for each webhook (last event status, etc) - return await WebhooksConfig( ).select( '*' ).where( { streamId } ) + let webhooks = await WebhooksConfig( ).select( '*' ).where( { streamId } ) + for ( let webhook of webhooks ) { + webhook.events = Object.keys( webhook.events ) + } + return webhooks + }, + + async getStreamWebhooksCount( { streamId } ) { + let [ res ] = await WebhooksConfig( ).count().where( { streamId } ) + return parseInt( res.count ) }, async dispatchStreamEvent( { streamId, event, eventPayload } ) { diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js index fa32d46ce..41a7f3664 100644 --- a/packages/server/modules/webhooks/tests/webhooks.spec.js +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -34,10 +34,7 @@ describe( 'Webhooks', ( ) => { description: 'test wh', secret: 'secret', enabled: true, - events: { - 'commit_create': true, - 'commit_update': true - } + events: [ 'commit_create', 'commit_update' ] } before( async ( ) => { @@ -101,5 +98,32 @@ describe( 'Webhooks', ( ) => { expect( lastEvents ).to.have.lengthOf( 1 ) expect( lastEvents[0].payload ).to.equal( 'payload123' ) } ) + + it( 'Should have a webhook limit for streams', async ( ) => { + let limit = 100 + for ( let i = 0; i < limit - 1; i++ ) { + await createWebhook( webhookOne ) + } + try { + await createWebhook( webhookOne ) + } catch ( err ) { + if ( err.toString().indexOf( 'Maximum' ) > -1 ) return + } + assert.fail( 'Configured more webhooks than the limit' ) + } ) + + it( 'Should cleanup stream webhooks', async ( ) => { + // just cleanup the 99 extra webhooks added before (not a real test) + let streamWebhooks = await getStreamWebhooks( { streamId: streamOne.id } ) + for ( let webhook of streamWebhooks ) { + if ( webhook.id != webhookOne.id ) { + await deleteWebhook( { id: webhook.id } ) + } + } + streamWebhooks = await getStreamWebhooks( { streamId: streamOne.id } ) + expect( streamWebhooks ).to.have.lengthOf( 1 ) + expect( streamWebhooks[0] ).to.have.property( 'id' ) + expect( streamWebhooks[0].id ).to.equal( webhookOne.id ) + } ) } ) } ) diff --git a/packages/webhook-service/package.json b/packages/webhook-service/package.json index 1289b2bcc..b72b20d84 100644 --- a/packages/webhook-service/package.json +++ b/packages/webhook-service/package.json @@ -12,7 +12,8 @@ }, "homepage": "https://github.com/specklesystems/speckle-server#readme", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "dev": "node src/main.js" }, "dependencies": { "knex": "^0.95.7", From 2bb27524a4d2fcebe9827e89821d8da7a3977f90 Mon Sep 17 00:00:00 2001 From: cristi8 Date: Tue, 13 Jul 2021 11:16:15 +0300 Subject: [PATCH 08/21] renamed webhooks_config:events column to triggers --- .../webhooks/graph/resolvers/webhooks.js | 4 ++-- .../webhooks/graph/schemas/webhooks.graphql | 6 +++--- .../migrations/20210701180000-webhooks.js | 2 +- .../modules/webhooks/services/webhooks.js | 20 +++++++++---------- .../modules/webhooks/tests/webhooks.spec.js | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/server/modules/webhooks/graph/resolvers/webhooks.js b/packages/server/modules/webhooks/graph/resolvers/webhooks.js index c70dd9c20..2c118b25a 100644 --- a/packages/server/modules/webhooks/graph/resolvers/webhooks.js +++ b/packages/server/modules/webhooks/graph/resolvers/webhooks.js @@ -26,12 +26,12 @@ module.exports = { Mutation: { async webhookCreate( parent, args, context, info ) { - let id = await createWebhook( { streamId: args.webhook.streamId, url: args.webhook.url, description: args.webhook.description, secret: args.webhook.secret, enabled: args.webhook.enabled !== false, events: args.webhook.events } ) + let id = await createWebhook( { streamId: args.webhook.streamId, url: args.webhook.url, description: args.webhook.description, secret: args.webhook.secret, enabled: args.webhook.enabled !== false, triggers: args.webhook.triggers } ) return id }, async webhookUpdate( parent, args, context, info ) { - let updated = await updateWebhook( { id: args.webhook.id, url: args.webhook.url, description: args.webhook.description, secret: args.webhook.secret, enabled: args.webhook.enabled !== false, events: args.webhook.events } ) + let updated = await updateWebhook( { id: args.webhook.id, url: args.webhook.url, description: args.webhook.description, secret: args.webhook.secret, enabled: args.webhook.enabled !== false, triggers: args.webhook.triggers } ) return !!updated }, diff --git a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql index 163fb8522..09a335d03 100644 --- a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql +++ b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql @@ -38,7 +38,7 @@ type Webhook { streamId: String! url: String! description: String - events: [String]! + triggers: [String]! enabled: Boolean history(limit: Int! = 25): WebhookEventCollection } @@ -47,7 +47,7 @@ input WebhookCreateInput { streamId: String! url: String! description: String - events: [String]! + triggers: [String]! secret: String enabled: Boolean } @@ -58,7 +58,7 @@ input WebhookUpdateInput { description: String secret: String enabled: Boolean - events: [String] + triggers: [String] } type WebhookEventCollection{ diff --git a/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js b/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js index fcbd82a3e..2df9dad88 100644 --- a/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js +++ b/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js @@ -7,7 +7,7 @@ exports.up = async knex => { table.string( 'streamId', 10 ).references( 'id' ).inTable( 'streams' ).onDelete( 'cascade' ) table.string( 'url' ) table.string( 'description' ) - table.jsonb( 'events' ) + table.jsonb( 'triggers' ) table.string( 'secret' ) table.boolean( 'enabled' ).defaultTo( true ) diff --git a/packages/server/modules/webhooks/services/webhooks.js b/packages/server/modules/webhooks/services/webhooks.js index a6c3c0807..1f6862484 100644 --- a/packages/server/modules/webhooks/services/webhooks.js +++ b/packages/server/modules/webhooks/services/webhooks.js @@ -11,13 +11,13 @@ let MAX_STREAM_WEBHOOKS = 100 module.exports = { - async createWebhook( { streamId, url, description, secret, enabled, events } ) { + async createWebhook( { streamId, url, description, secret, enabled, triggers } ) { let streamWebhookCount = await module.exports.getStreamWebhooksCount( { streamId } ) if ( streamWebhookCount >= MAX_STREAM_WEBHOOKS ) { throw new Error( `Maximum number of webhooks for a stream reached (${MAX_STREAM_WEBHOOKS})` ) } - let eventsObj = Object.assign( {}, ...events.map( ( x ) => ( { [x]: true } ) ) ) + let triggersObj = Object.assign( {}, ...triggers.map( ( x ) => ( { [x]: true } ) ) ) let [ id ] = await WebhooksConfig( ).returning( 'id' ).insert( { id: crs( { length: 10 } ), @@ -26,7 +26,7 @@ module.exports = { description, secret, enabled, - events: eventsObj + triggers: triggersObj } ) return id }, @@ -34,20 +34,20 @@ module.exports = { async getWebhook( { id } ) { let webhook = await WebhooksConfig().select( '*' ).where( { id } ).first() if ( webhook ) { - webhook.events = Object.keys( webhook.events ) + webhook.triggers = Object.keys( webhook.triggers ) } return webhook }, - async updateWebhook( { id, url, description, secret, enabled, events } ) { + async updateWebhook( { id, url, description, secret, enabled, triggers } ) { let fieldsToUpdate = {} if ( url !== undefined ) fieldsToUpdate.url = url if ( description !== undefined ) fieldsToUpdate.description = description if ( secret !== undefined ) fieldsToUpdate.secret = secret if ( enabled !== undefined ) fieldsToUpdate.enabled = enabled - if ( events !== undefined ) { - let eventsObj = Object.assign( {}, ...events.map( ( x ) => ( { [x]: true } ) ) ) - fieldsToUpdate.events = eventsObj + if ( triggers !== undefined ) { + let triggersObj = Object.assign( {}, ...triggers.map( ( x ) => ( { [x]: true } ) ) ) + fieldsToUpdate.triggers = triggersObj } let [ res ] = await WebhooksConfig( ) @@ -64,7 +64,7 @@ module.exports = { async getStreamWebhooks( { streamId } ) { let webhooks = await WebhooksConfig( ).select( '*' ).where( { streamId } ) for ( let webhook of webhooks ) { - webhook.events = Object.keys( webhook.events ) + webhook.triggers = Object.keys( webhook.triggers ) } return webhooks }, @@ -79,7 +79,7 @@ module.exports = { SELECT * FROM webhooks_config WHERE "streamId" = ? `, [ streamId ] ) for ( let wh of rows ) { - if ( !( event in wh.events ) ) + if ( !( event in wh.triggers ) ) continue await WebhooksEvents( ).insert( { diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js index 41a7f3664..bbb1e2721 100644 --- a/packages/server/modules/webhooks/tests/webhooks.spec.js +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -34,7 +34,7 @@ describe( 'Webhooks', ( ) => { description: 'test wh', secret: 'secret', enabled: true, - events: [ 'commit_create', 'commit_update' ] + triggers: [ 'commit_create', 'commit_update' ] } before( async ( ) => { From 9a379421382e1db03e50c770e28110e76266cfee Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Mon, 12 Jul 2021 18:31:55 +0100 Subject: [PATCH 09/21] test(webhooks): add gql tests --- .../modules/webhooks/tests/webhooks.spec.js | 150 ++++++++++++++++-- 1 file changed, 137 insertions(+), 13 deletions(-) diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js index bbb1e2721..2ffc2fe1e 100644 --- a/packages/server/modules/webhooks/tests/webhooks.spec.js +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -4,18 +4,22 @@ const chaiHttp = require( 'chai-http' ) const assert = require( 'assert' ) const appRoot = require( 'app-root-path' ) -const { init } = require( `${appRoot}/app` ) +const { init, startHttp } = require( `${appRoot}/app` ) const knex = require( `${appRoot}/db/knex` ) +const { createPersonalAccessToken } = require( '../../core/services/tokens' ) +const { createWebhook, getStreamWebhooks, getLastWebhookEvents, getWebhook, updateWebhook, deleteWebhook, dispatchStreamEvent } = require( '../services/webhooks' ) +const { createUser } = require( '../../core/services/users' ) +const { createStream, getStream, grantPermissionsStream } = require( '../../core/services/streams' ) const expect = chai.expect chai.use( chaiHttp ) +const serverAddress = `http://localhost:${process.env.PORT || 3000}` -const { createWebhook, getStreamWebhooks, getLastWebhookEvents, getWebhook, updateWebhook, deleteWebhook, dispatchStreamEvent } = require( '../services/webhooks' ) -const { createUser } = require( '../../core/services/users' ) -const { createStream, getStream } = require( '../../core/services/streams' ) -describe( 'Webhooks', ( ) => { +describe( 'Webhooks @webhooks', () => { + let testServer + let userOne = { name: 'User', email: 'user@gmail.com', @@ -23,7 +27,7 @@ describe( 'Webhooks', ( ) => { } let streamOne = { - name: 'stream', + name: 'streamOne', description: 'stream', isPublic: true } @@ -40,16 +44,20 @@ describe( 'Webhooks', ( ) => { before( async ( ) => { await knex.migrate.rollback( ) await knex.migrate.latest( ) - await init() + let { app } = await init() + let { server } = await startHttp( app ) + testServer = server userOne.id = await createUser( userOne ) streamOne.ownerId = userOne.id streamOne.id = await createStream( streamOne ) + webhookOne.streamId = streamOne.id } ) after( async ( ) => { // await knex.migrate.rollback( ) + testServer.close( ) } ) describe( 'Create, Read, Update, Delete Webhooks', ( ) => { @@ -81,22 +89,118 @@ describe( 'Webhooks', ( ) => { expect( webhook ).to.be.undefined } ) - it( 'Should get webooks for stream', async ( ) => { + it( 'Should get webhooks for stream', async ( ) => { let streamWebhooks = await getStreamWebhooks( { streamId: streamOne.id } ) expect( streamWebhooks ).to.have.lengthOf( 0 ) - + webhookOne.id = await createWebhook( webhookOne ) streamWebhooks = await getStreamWebhooks( { streamId: streamOne.id } ) expect( streamWebhooks ).to.have.lengthOf( 1 ) - expect( streamWebhooks[0] ).to.have.property( 'url' ) - expect( streamWebhooks[0].url ).to.equal( webhookOne.url ) + expect( streamWebhooks[ 0 ] ).to.have.property( 'url' ) + expect( streamWebhooks[ 0 ].url ).to.equal( webhookOne.url ) } ) - it( 'Should dispatch and get events', async () => { + it( 'Should dispatch and get events', async () => { await dispatchStreamEvent( { streamId: streamOne.id, event: 'commit_create', eventPayload: 'payload123' } ) let lastEvents = await getLastWebhookEvents( { webhookId: webhookOne.id } ) expect( lastEvents ).to.have.lengthOf( 1 ) - expect( lastEvents[0].payload ).to.equal( 'payload123' ) + expect( lastEvents[ 0 ].payload ).to.equal( 'payload123' ) + } ) + } ) + + describe( 'GraphQL API Webhooks @webhooks-api', () => { + let userTwo = { + name: 'User2', + email: 'user2@gmail.com', + password: 'jdsadjsadasfdsa' + } + + let webhookTwo = { + streamId: null, + url: 'http://localhost:42/non-existent-two', + description: 'test wh no 2', + secret: 'secret', + enabled: true, + events: { + 'commit_create': true, + 'commit_update': true + } + } + + let streamTwo = { + name: 'streamTwo', + description: 'stream', + isPublic: true + } + + before( async () => { + userTwo.id = await createUser( userTwo ) + streamTwo.ownerId = userOne.id + streamTwo.id = await createStream( streamTwo ) + webhookTwo.streamId = streamTwo.id + + + userOne.token = `Bearer ${( await createPersonalAccessToken( userOne.id, 'userOne test token', [ 'streams:read', 'streams:write' ] ) )}` + userTwo.token = `Bearer ${( await createPersonalAccessToken( userTwo.id, 'userTwo test token', [ 'streams:read', 'streams:write' ] ) )}` + await grantPermissionsStream( { streamId: streamTwo.id, userId: userTwo.id, role: 'stream:contributor' } ) + } ) + + it( 'Should create a webhook', async () => { + const res = await sendRequest( userOne.token, { query: 'mutation createWebhook($webhook: WebhookCreateInput!) { webhookCreate( webhook: $webhook ) }', variables: { webhook: webhookTwo } } ) + expect( noErrors( res ) ) + expect( res.body.data.webhookCreate ).to.not.be.null + webhookTwo.id = res.body.data.webhookCreate + } ) + + it( 'Should get stream webhooks and the previous events', async () => { + await dispatchStreamEvent( { streamId: streamTwo.id, event: 'commit_create', eventPayload: 'payload321' } ) + const res = await sendRequest( userOne.token, { query: `query { + stream(id: "${streamTwo.id}") { + webhooks { totalCount items { id url enabled + history { totalCount items { status statusInfo payload } } } + } + } + }` } ) + expect( noErrors( res ) ) + let webhooks = res.body.data.stream.webhooks + + expect( webhooks.totalCount ).to.equal( 1 ) + expect( webhooks.items[ 0 ].url ).to.equal( webhookTwo.url ) + expect( webhooks.items[ 0 ].history.totalCount ).to.equal( 1 ) + expect( webhooks.items[ 0 ].history.items[ 0 ].payload ).to.equal( 'payload321' ) + } ) + + it( 'Should update a webhook', async () => { + const res = await sendRequest( userOne.token, { query: `mutation { webhookUpdate(webhook: { id: "${webhookTwo.id}", description: "updated webhook", enabled: false }) + }` } ) + let webhook = await getWebhook( { id: webhookTwo.id } ) + expect( noErrors( res ) ) + expect( res.body.data.webhookUpdate ).to.equal( 'true' ) + expect( webhook.description ).to.equal( 'updated webhook' ) + expect( webhook.enabled ).to.equal( false ) + } ) + + it( 'Should delete a webhook', async () => { + const res = await sendRequest( userOne.token, { + query: `mutation { webhookDelete(id: "${webhookTwo.id}") }` + } ) + expect( noErrors( res ) ) + expect( res.body.data.webhookDelete ).to.equal( 'true' ) + } ) + + it( 'Should *not* create a webhook if user is not a stream owner', async () => { + delete webhookTwo.id + const res = await sendRequest( userTwo.token, { query: 'mutation createWebhook($webhook: WebhookCreateInput!) { webhookCreate( webhook: $webhook ) }', variables: { webhook: webhookTwo } } ) + expect( res.body.errors ).to.exist + expect( res.body.errors[ 0 ].extensions.code ).to.equal( 'FORBIDDEN' ) + } ) + + it( 'Should *not* get a webhook if the user is not a stream owner', async () => { + const res = await sendRequest( userTwo.token, { query: `query { + stream(id: "${streamTwo.id}") { webhooks { totalCount items { id url enabled } } } + }` } ) + expect( res.body.errors ).to.exist + expect( res.body.errors[ 0 ].extensions.code ).to.equal( 'FORBIDDEN' ) } ) it( 'Should have a webhook limit for streams', async ( ) => { @@ -127,3 +231,23 @@ describe( 'Webhooks', ( ) => { } ) } ) } ) + + +/** + * Sends a graphql request. Convenience wrapper. + * @param {string} auth the user's token + * @param {string} obj the query/mutation to send + * @return {Promise} the awaitable request + */ +function sendRequest( auth, obj, address = serverAddress ) { + return chai.request( address ).post( '/graphql' ).set( 'Authorization', auth ).send( obj ) +} + +/** + * Checks the response body for errors. To be used in expect assertions. + * Will throw an error if 'errors' exist. + * @param {*} res + */ +function noErrors( res ) { + if ( 'errors' in res.body ) throw new Error( `Failed GraphQL request: ${res.body.errors[ 0 ].message}` ) +} From 259c0b8e5d5130bdae2a1cdc077e8d9b5124e655 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 13 Jul 2021 09:21:34 +0100 Subject: [PATCH 10/21] test(webhooks): integrate changes --- .../server/modules/webhooks/tests/webhooks.spec.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js index 2ffc2fe1e..89b2d5e24 100644 --- a/packages/server/modules/webhooks/tests/webhooks.spec.js +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -121,10 +121,7 @@ describe( 'Webhooks @webhooks', () => { description: 'test wh no 2', secret: 'secret', enabled: true, - events: { - 'commit_create': true, - 'commit_update': true - } + triggers: [ 'commit_create', 'commit_update' ] } let streamTwo = { @@ -208,11 +205,13 @@ describe( 'Webhooks @webhooks', () => { for ( let i = 0; i < limit - 1; i++ ) { await createWebhook( webhookOne ) } + try { await createWebhook( webhookOne ) } catch ( err ) { if ( err.toString().indexOf( 'Maximum' ) > -1 ) return } + assert.fail( 'Configured more webhooks than the limit' ) } ) @@ -224,10 +223,11 @@ describe( 'Webhooks @webhooks', () => { await deleteWebhook( { id: webhook.id } ) } } + streamWebhooks = await getStreamWebhooks( { streamId: streamOne.id } ) expect( streamWebhooks ).to.have.lengthOf( 1 ) - expect( streamWebhooks[0] ).to.have.property( 'id' ) - expect( streamWebhooks[0].id ).to.equal( webhookOne.id ) + expect( streamWebhooks[ 0 ] ).to.have.property( 'id' ) + expect( streamWebhooks[ 0 ].id ).to.equal( webhookOne.id ) } ) } ) } ) From e7a48adbb27dee7347a27b739a47e7c784bbb050 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 13 Jul 2021 09:32:00 +0100 Subject: [PATCH 11/21] test(webhooks): try a diff port? --- packages/server/modules/webhooks/tests/webhooks.spec.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js index 89b2d5e24..bda81c7ae 100644 --- a/packages/server/modules/webhooks/tests/webhooks.spec.js +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -14,7 +14,8 @@ const { createStream, getStream, grantPermissionsStream } = require( '../../core const expect = chai.expect chai.use( chaiHttp ) -const serverAddress = `http://localhost:${process.env.PORT || 3000}` +const port = 3420 +const serverAddress = `http://localhost:${port}` describe( 'Webhooks @webhooks', () => { @@ -45,7 +46,7 @@ describe( 'Webhooks @webhooks', () => { await knex.migrate.rollback( ) await knex.migrate.latest( ) let { app } = await init() - let { server } = await startHttp( app ) + let { server } = await startHttp( app, port ) testServer = server userOne.id = await createUser( userOne ) From 26cb15251d63ef6e8d5d2661d310b974336459d8 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 13 Jul 2021 09:36:43 +0100 Subject: [PATCH 12/21] fix(webhooks): remove unuse field --- packages/server/modules/webhooks/graph/schemas/webhooks.graphql | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql index 09a335d03..befdf2e5d 100644 --- a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql +++ b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql @@ -29,7 +29,6 @@ extend type Mutation { type WebhookCollection { totalCount: Int - cursor: String items: [Webhook] } @@ -63,7 +62,6 @@ input WebhookUpdateInput { type WebhookEventCollection{ totalCount: Int - cursor: String items: [WebhookEvent] } From 32808437e578768bc576f438e0ea969909110cfa Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Fri, 16 Jul 2021 12:54:00 +0100 Subject: [PATCH 13/21] feat(frontend): webhooks / settings ui --- .../frontend/src/components/SidebarStream.vue | 9 +- .../src/components/admin/AdminCard.vue | 14 +- .../src/components/settings/WebhookForm.vue | 237 ++++++++++++++++++ packages/frontend/src/graphql/webhook.gql | 21 ++ packages/frontend/src/graphql/webhooks.gql | 21 ++ packages/frontend/src/router/index.js | 52 +++- .../src/views/settings/SettingsGeneral.vue | 18 ++ .../src/views/settings/SettingsWebhooks.vue | 139 ++++++++++ .../src/views/settings/StreamSettings.vue | 125 +++++++++ .../modules/webhooks/services/webhooks.js | 6 +- 10 files changed, 629 insertions(+), 13 deletions(-) create mode 100644 packages/frontend/src/components/settings/WebhookForm.vue create mode 100644 packages/frontend/src/graphql/webhook.gql create mode 100644 packages/frontend/src/graphql/webhooks.gql create mode 100644 packages/frontend/src/views/settings/SettingsGeneral.vue create mode 100644 packages/frontend/src/views/settings/SettingsWebhooks.vue create mode 100644 packages/frontend/src/views/settings/StreamSettings.vue diff --git a/packages/frontend/src/components/SidebarStream.vue b/packages/frontend/src/components/SidebarStream.vue index edde19ff4..a80be6ddd 100644 --- a/packages/frontend/src/components/SidebarStream.vue +++ b/packages/frontend/src/components/SidebarStream.vue @@ -98,7 +98,7 @@ class="px-0" @click="editStreamDialog = true" > - mdi-cog-outline + mdi-pencil-outline Edit @@ -125,6 +125,13 @@ + + + mdi-cog-outline + Settings + + +
Collaborators
 diff --git a/packages/frontend/src/components/admin/AdminCard.vue b/packages/frontend/src/components/admin/AdminCard.vue index d327c18e4..4232ee76f 100644 --- a/packages/frontend/src/components/admin/AdminCard.vue +++ b/packages/frontend/src/components/admin/AdminCard.vue @@ -1,11 +1,19 @@ + + You don't have any webhooks on this stream yet. Click the blue "Add Webhook" button in the top + right to add one. + + - {{ stream.name }} + + {{ stream.name }} + + mdi-chevron-left + back to stream + +
From fff075253178809b8fc78dbbecbff565858a2286 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Fri, 16 Jul 2021 16:14:28 +0100 Subject: [PATCH 15/21] feat(frontend): general stream details settings --- .../src/views/settings/SettingsGeneral.vue | 129 +++++++++++++++++- 1 file changed, 128 insertions(+), 1 deletion(-) diff --git a/packages/frontend/src/views/settings/SettingsGeneral.vue b/packages/frontend/src/views/settings/SettingsGeneral.vue index 708ce3d4a..f456bebfd 100644 --- a/packages/frontend/src/views/settings/SettingsGeneral.vue +++ b/packages/frontend/src/views/settings/SettingsGeneral.vue @@ -1,8 +1,73 @@ From 4f17a66866bea0a4b15dcb45658d86cd00234fb7 Mon Sep 17 00:00:00 2001 From: cristi8 Date: Tue, 20 Jul 2021 11:10:27 +0300 Subject: [PATCH 16/21] added id field on commit_create events, improved webhooks migration --- packages/server/modules/core/graph/resolvers/commits.js | 2 +- .../webhooks/migrations/20210701180000-webhooks.js | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/modules/core/graph/resolvers/commits.js b/packages/server/modules/core/graph/resolvers/commits.js index 29a21ad41..f098b3c39 100644 --- a/packages/server/modules/core/graph/resolvers/commits.js +++ b/packages/server/modules/core/graph/resolvers/commits.js @@ -86,7 +86,7 @@ module.exports = { resourceId: id, actionType: 'commit_create', userId: context.userId, - info: { commit: args.commit }, + info: { id: id, commit: args.commit }, message: `Commit created on branch ${args.commit.branchName}: ${id} (${args.commit.message})` } ) await pubsub.publish( COMMIT_CREATED, { diff --git a/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js b/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js index 2df9dad88..8d4654ac5 100644 --- a/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js +++ b/packages/server/modules/webhooks/migrations/20210701180000-webhooks.js @@ -5,8 +5,8 @@ exports.up = async knex => { await knex.schema.createTable( 'webhooks_config', table => { table.string( 'id' ).primary( ) table.string( 'streamId', 10 ).references( 'id' ).inTable( 'streams' ).onDelete( 'cascade' ) - table.string( 'url' ) - table.string( 'description' ) + table.text( 'url' ) + table.text( 'description' ) table.jsonb( 'triggers' ) table.string( 'secret' ) table.boolean( 'enabled' ).defaultTo( true ) @@ -19,11 +19,11 @@ exports.up = async knex => { table.string( 'webhookId' ).references( 'id' ).inTable( 'webhooks_config' ).onDelete( 'cascade' ) table.integer( 'status' ).notNullable( ).defaultTo( 0 ) - table.string( 'statusInfo' ).notNullable( ).defaultTo( 'Pending' ) + table.text( 'statusInfo' ).notNullable( ).defaultTo( 'Pending' ) table.timestamp( 'lastUpdate' ).notNullable( ).defaultTo( knex.fn.now( ) ) - table.string( 'payload' ) + table.text( 'payload' ) table.index( 'webhookId' ) table.index( 'status' ) From eb829d9ded678c07fee805992d2ceb7fd1571bc0 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 20 Jul 2021 16:39:22 +0100 Subject: [PATCH 17/21] fix(server): webhook permissions fixes --- .../webhooks/graph/resolvers/webhooks.js | 12 +++++++++- .../webhooks/graph/schemas/webhooks.graphql | 16 +++++++++----- .../modules/webhooks/tests/webhooks.spec.js | 22 +++++++++++-------- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/packages/server/modules/webhooks/graph/resolvers/webhooks.js b/packages/server/modules/webhooks/graph/resolvers/webhooks.js index 2c118b25a..943c9a954 100644 --- a/packages/server/modules/webhooks/graph/resolvers/webhooks.js +++ b/packages/server/modules/webhooks/graph/resolvers/webhooks.js @@ -1,9 +1,13 @@ +const appRoot = require( 'app-root-path' ) +const { authorizeResolver } = require( `${appRoot}/modules/shared` ) const { createWebhook, getWebhook, updateWebhook, deleteWebhook, getStreamWebhooks, getLastWebhookEvents, getWebhookEventsCount } = require( '../../services/webhooks' ) module.exports = { Stream: { async webhooks( parent, args, context, info ) { + await authorizeResolver( context.userId, parent.id, 'stream:owner' ) + if ( args.id ) { let wh = await getWebhook( { id: args.id } ) let items = wh ? [ wh ] : [] @@ -26,17 +30,23 @@ module.exports = { Mutation: { async webhookCreate( parent, args, context, info ) { + await authorizeResolver( context.userId, args.webhook.streamId, 'stream:owner' ) + let id = await createWebhook( { streamId: args.webhook.streamId, url: args.webhook.url, description: args.webhook.description, secret: args.webhook.secret, enabled: args.webhook.enabled !== false, triggers: args.webhook.triggers } ) return id }, async webhookUpdate( parent, args, context, info ) { + await authorizeResolver( context.userId, args.webhook.streamId, 'stream:owner' ) + let updated = await updateWebhook( { id: args.webhook.id, url: args.webhook.url, description: args.webhook.description, secret: args.webhook.secret, enabled: args.webhook.enabled !== false, triggers: args.webhook.triggers } ) return !!updated }, async webhookDelete( parent, args, context, info ) { - let deleted = await deleteWebhook( { id: args.id } ) + await authorizeResolver( context.userId, args.webhook.streamId, 'stream:owner' ) + + let deleted = await deleteWebhook( { id: args.webhook.id } ) return !!deleted } diff --git a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql index befdf2e5d..8ec9e4f12 100644 --- a/packages/server/modules/webhooks/graph/schemas/webhooks.graphql +++ b/packages/server/modules/webhooks/graph/schemas/webhooks.graphql @@ -1,6 +1,6 @@ extend type Stream { webhooks(id: String): WebhookCollection - @hasRole(role: "stream:owner") + @hasRole(role: "server:user") @hasScope(scope: "streams:write") } @@ -9,21 +9,21 @@ extend type Mutation { Creates a new webhook on a stream """ webhookCreate(webhook: WebhookCreateInput!): String! - @hasRole(role: "stream:owner") + @hasRole(role: "server:user") @hasScope(scope: "streams:write") """ Updates an existing webhook """ webhookUpdate(webhook: WebhookUpdateInput!): String! - @hasRole(role: "stream:owner") + @hasRole(role: "server:user") @hasScope(scope: "streams:write") """ Deletes an existing webhook """ - webhookDelete(id: String!): String! - @hasRole(role: "stream:owner") + webhookDelete(webhook: WebhookDeleteInput!): String! + @hasRole(role: "server:user") @hasScope(scope: "streams:write") } @@ -53,6 +53,7 @@ input WebhookCreateInput { input WebhookUpdateInput { id: String! + streamId: String! url: String description: String secret: String @@ -60,6 +61,11 @@ input WebhookUpdateInput { triggers: [String] } +input WebhookDeleteInput { + id: String! + streamId: String! +} + type WebhookEventCollection{ totalCount: Int items: [WebhookEvent] diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js index bda81c7ae..21473dee1 100644 --- a/packages/server/modules/webhooks/tests/webhooks.spec.js +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -133,18 +133,18 @@ describe( 'Webhooks @webhooks', () => { before( async () => { userTwo.id = await createUser( userTwo ) - streamTwo.ownerId = userOne.id + streamTwo.ownerId = userTwo.id streamTwo.id = await createStream( streamTwo ) webhookTwo.streamId = streamTwo.id userOne.token = `Bearer ${( await createPersonalAccessToken( userOne.id, 'userOne test token', [ 'streams:read', 'streams:write' ] ) )}` userTwo.token = `Bearer ${( await createPersonalAccessToken( userTwo.id, 'userTwo test token', [ 'streams:read', 'streams:write' ] ) )}` - await grantPermissionsStream( { streamId: streamTwo.id, userId: userTwo.id, role: 'stream:contributor' } ) + await grantPermissionsStream( { streamId: streamTwo.id, userId: userOne.id, role: 'stream:contributor' } ) } ) it( 'Should create a webhook', async () => { - const res = await sendRequest( userOne.token, { query: 'mutation createWebhook($webhook: WebhookCreateInput!) { webhookCreate( webhook: $webhook ) }', variables: { webhook: webhookTwo } } ) + const res = await sendRequest( userTwo.token, { query: 'mutation createWebhook($webhook: WebhookCreateInput!) { webhookCreate( webhook: $webhook ) }', variables: { webhook: webhookTwo } } ) expect( noErrors( res ) ) expect( res.body.data.webhookCreate ).to.not.be.null webhookTwo.id = res.body.data.webhookCreate @@ -152,7 +152,7 @@ describe( 'Webhooks @webhooks', () => { it( 'Should get stream webhooks and the previous events', async () => { await dispatchStreamEvent( { streamId: streamTwo.id, event: 'commit_create', eventPayload: 'payload321' } ) - const res = await sendRequest( userOne.token, { query: `query { + const res = await sendRequest( userTwo.token, { query: `query { stream(id: "${streamTwo.id}") { webhooks { totalCount items { id url enabled history { totalCount items { status statusInfo payload } } } @@ -169,7 +169,8 @@ describe( 'Webhooks @webhooks', () => { } ) it( 'Should update a webhook', async () => { - const res = await sendRequest( userOne.token, { query: `mutation { webhookUpdate(webhook: { id: "${webhookTwo.id}", description: "updated webhook", enabled: false }) + const res = await sendRequest( userTwo.token, { + query: `mutation { webhookUpdate(webhook: { id: "${webhookTwo.id}", streamId: "${streamTwo.id}", description: "updated webhook", enabled: false }) }` } ) let webhook = await getWebhook( { id: webhookTwo.id } ) expect( noErrors( res ) ) @@ -179,8 +180,8 @@ describe( 'Webhooks @webhooks', () => { } ) it( 'Should delete a webhook', async () => { - const res = await sendRequest( userOne.token, { - query: `mutation { webhookDelete(id: "${webhookTwo.id}") }` + const res = await sendRequest( userTwo.token, { + query: `mutation { webhookDelete(webhook: { id: "${webhookTwo.id}", streamId: "${streamTwo.id}" } ) }` } ) expect( noErrors( res ) ) expect( res.body.data.webhookDelete ).to.equal( 'true' ) @@ -188,13 +189,16 @@ describe( 'Webhooks @webhooks', () => { it( 'Should *not* create a webhook if user is not a stream owner', async () => { delete webhookTwo.id - const res = await sendRequest( userTwo.token, { query: 'mutation createWebhook($webhook: WebhookCreateInput!) { webhookCreate( webhook: $webhook ) }', variables: { webhook: webhookTwo } } ) + const res = await sendRequest( userOne.token, { + query: 'mutation createWebhook($webhook: WebhookCreateInput!) { webhookCreate( webhook: $webhook ) }', + variables: { webhook: webhookTwo } + } ) expect( res.body.errors ).to.exist expect( res.body.errors[ 0 ].extensions.code ).to.equal( 'FORBIDDEN' ) } ) it( 'Should *not* get a webhook if the user is not a stream owner', async () => { - const res = await sendRequest( userTwo.token, { query: `query { + const res = await sendRequest( userOne.token, { query: `query { stream(id: "${streamTwo.id}") { webhooks { totalCount items { id url enabled } } } }` } ) expect( res.body.errors ).to.exist From 34b78f07f7756fb034f1071d5f7724dfc90b4cdd Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Tue, 20 Jul 2021 16:39:54 +0100 Subject: [PATCH 18/21] fix(frontend): webhooks routing and ui --- packages/frontend/src/components/SidebarStream.vue | 2 +- .../frontend/src/components/settings/WebhookForm.vue | 7 ++++--- packages/frontend/src/router/index.js | 2 +- .../frontend/src/views/settings/SettingsWebhooks.vue | 9 +++++++-- packages/frontend/src/views/settings/StreamSettings.vue | 7 ++++--- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/frontend/src/components/SidebarStream.vue b/packages/frontend/src/components/SidebarStream.vue index a80be6ddd..4c3db7ea5 100644 --- a/packages/frontend/src/components/SidebarStream.vue +++ b/packages/frontend/src/components/SidebarStream.vue @@ -126,7 +126,7 @@ - + mdi-cog-outline Settings diff --git a/packages/frontend/src/components/settings/WebhookForm.vue b/packages/frontend/src/components/settings/WebhookForm.vue index d0d611d6d..3d502be50 100644 --- a/packages/frontend/src/components/settings/WebhookForm.vue +++ b/packages/frontend/src/components/settings/WebhookForm.vue @@ -166,6 +166,7 @@ export default { let params = { id: this.webhook.id, + streamId: this.streamId, url: this.url, description: this.description, triggers: this.triggers, @@ -219,12 +220,12 @@ export default { await this.$apollo.mutate({ mutation: gql` - mutation webhookDelete($id: String!) { - webhookDelete(id: $id) + mutation webhookDelete($params: WebhookDeleteInput!) { + webhookDelete(webhook: $params) } `, variables: { - id: this.webhookId + params: { id: this.webhookId, streamId: this.streamId } } }) diff --git a/packages/frontend/src/router/index.js b/packages/frontend/src/router/index.js index 15b8565cd..a5228762c 100644 --- a/packages/frontend/src/router/index.js +++ b/packages/frontend/src/router/index.js @@ -142,7 +142,7 @@ const routes = [ ] }, { - path: 'settings/:streamId/', + path: 'settings/streams/:streamId/', name: 'settings', props: true, component: () => import('../views/settings/StreamSettings.vue'), diff --git a/packages/frontend/src/views/settings/SettingsWebhooks.vue b/packages/frontend/src/views/settings/SettingsWebhooks.vue index 8d179c327..aeae185b4 100644 --- a/packages/frontend/src/views/settings/SettingsWebhooks.vue +++ b/packages/frontend/src/views/settings/SettingsWebhooks.vue @@ -22,7 +22,12 @@ @@ -36,7 +41,7 @@ diff --git a/packages/frontend/src/views/settings/StreamSettings.vue b/packages/frontend/src/views/settings/StreamSettings.vue index 3feca1070..2a1d610b1 100644 --- a/packages/frontend/src/views/settings/StreamSettings.vue +++ b/packages/frontend/src/views/settings/StreamSettings.vue @@ -3,8 +3,9 @@ - + {{ stream.name }} +
mdi-chevron-left back to stream @@ -73,11 +74,11 @@ export default { childRoutes: [ { name: 'General', - to: `/settings/${this.$attrs.streamId}/general` + to: `/settings/streams/${this.$attrs.streamId}/general` }, { name: 'Webhooks', - to: `/settings/${this.$attrs.streamId}/webhooks` + to: `/settings/streams/${this.$attrs.streamId}/webhooks` } ] } From 1adb39f02b49dcdec1fd7ab13a1114954f03e205 Mon Sep 17 00:00:00 2001 From: izzy lyseggen Date: Wed, 21 Jul 2021 18:44:09 +0100 Subject: [PATCH 19/21] fix(server): webhooks mutations tests forgot to push this commit before merge ah sorry!!! --- .../webhooks/graph/resolvers/webhooks.js | 10 ++++++++++ .../modules/webhooks/tests/webhooks.spec.js | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/server/modules/webhooks/graph/resolvers/webhooks.js b/packages/server/modules/webhooks/graph/resolvers/webhooks.js index 943c9a954..f34948975 100644 --- a/packages/server/modules/webhooks/graph/resolvers/webhooks.js +++ b/packages/server/modules/webhooks/graph/resolvers/webhooks.js @@ -1,4 +1,6 @@ const appRoot = require( 'app-root-path' ) +const { ForbiddenError } = require( 'apollo-server-express' ) + const { authorizeResolver } = require( `${appRoot}/modules/shared` ) const { createWebhook, getWebhook, updateWebhook, deleteWebhook, getStreamWebhooks, getLastWebhookEvents, getWebhookEventsCount } = require( '../../services/webhooks' ) @@ -39,6 +41,10 @@ module.exports = { async webhookUpdate( parent, args, context, info ) { await authorizeResolver( context.userId, args.webhook.streamId, 'stream:owner' ) + let wh = await getWebhook( { id: args.webhook.id } ) + if ( args.webhook.streamId !== wh.streamId ) + throw new ForbiddenError( 'The webhook id and stream id do not match. Please check your inputs.' ) + let updated = await updateWebhook( { id: args.webhook.id, url: args.webhook.url, description: args.webhook.description, secret: args.webhook.secret, enabled: args.webhook.enabled !== false, triggers: args.webhook.triggers } ) return !!updated @@ -46,6 +52,10 @@ module.exports = { async webhookDelete( parent, args, context, info ) { await authorizeResolver( context.userId, args.webhook.streamId, 'stream:owner' ) + let wh = await getWebhook( { id: args.webhook.id } ) + if ( args.webhook.streamId !== wh.streamId ) + throw new ForbiddenError( 'The webhook id and stream id do not match. Please check your inputs.' ) + let deleted = await deleteWebhook( { id: args.webhook.id } ) return !!deleted diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js index 21473dee1..37eb3b798 100644 --- a/packages/server/modules/webhooks/tests/webhooks.spec.js +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -179,6 +179,22 @@ describe( 'Webhooks @webhooks', () => { expect( webhook.enabled ).to.equal( false ) } ) + it( 'Should *not* update or delete a webhook if the stream id and webhook id do not match', async () => { + const res1 = await sendRequest( userOne.token, { + query: `mutation { webhookDelete(webhook: { id: "${webhookTwo.id}", streamId: "${streamOne.id}" } ) }` + } ) + expect( res1.body.errors ).to.exist + expect( res1.body.errors[ 0 ].message ).to.equal( 'The webhook id and stream id do not match. Please check your inputs.' ) + expect( res1.body.errors[ 0 ].extensions.code ).to.equal( 'FORBIDDEN' ) + + const res2 = await sendRequest( userOne.token, { + query: `mutation { webhookUpdate(webhook: { id: "${webhookTwo.id}", streamId: "${streamOne.id}", description: "updated webhook", enabled: false }) }` + } ) + expect( res2.body.errors ).to.exist + expect( res2.body.errors[ 0 ].message ).to.equal( 'The webhook id and stream id do not match. Please check your inputs.' ) + expect( res2.body.errors[ 0 ].extensions.code ).to.equal( 'FORBIDDEN' ) + } ) + it( 'Should delete a webhook', async () => { const res = await sendRequest( userTwo.token, { query: `mutation { webhookDelete(webhook: { id: "${webhookTwo.id}", streamId: "${streamTwo.id}" } ) }` @@ -205,6 +221,7 @@ describe( 'Webhooks @webhooks', () => { expect( res.body.errors[ 0 ].extensions.code ).to.equal( 'FORBIDDEN' ) } ) + it( 'Should have a webhook limit for streams', async ( ) => { let limit = 100 for ( let i = 0; i < limit - 1; i++ ) { From 157db23f458b4a0c9c1e77ca6762fe827162b0fc Mon Sep 17 00:00:00 2001 From: cristi8 Date: Wed, 21 Jul 2021 20:45:57 +0300 Subject: [PATCH 20/21] Added server info and user info to webhook payload --- .../modules/activitystream/services/index.js | 9 ++++-- .../modules/webhooks/services/webhooks.js | 32 ++++++++++++++++++- packages/webhook-service/src/main.js | 15 +++------ 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/packages/server/modules/activitystream/services/index.js b/packages/server/modules/activitystream/services/index.js index 908eb5fa0..903394df3 100644 --- a/packages/server/modules/activitystream/services/index.js +++ b/packages/server/modules/activitystream/services/index.js @@ -22,8 +22,13 @@ module.exports = { await StreamActivity( ).insert( dbObject ) if ( streamId ) { let webhooksPayload = { - 'event_name': actionType, - 'data': info + streamId: streamId, + userId: userId, + activityMessage: message, + event: { + 'event_name': actionType, + 'data': info + } } dispatchStreamEvent( { streamId, event: actionType, eventPayload: webhooksPayload } ) } diff --git a/packages/server/modules/webhooks/services/webhooks.js b/packages/server/modules/webhooks/services/webhooks.js index bd6ba5639..05f6c1f3f 100644 --- a/packages/server/modules/webhooks/services/webhooks.js +++ b/packages/server/modules/webhooks/services/webhooks.js @@ -6,6 +6,10 @@ const crs = require( 'crypto-random-string' ) const WebhooksConfig = ( ) => knex( 'webhooks_config' ) const WebhooksEvents = ( ) => knex( 'webhooks_events' ) +const Users = ( ) => knex( 'users' ) + +const { getServerInfo } = require( '../../core/services/generic' ) +const { getStream } = require( '../../core/services/streams' ) let MAX_STREAM_WEBHOOKS = 100 @@ -77,17 +81,43 @@ module.exports = { }, async dispatchStreamEvent( { streamId, event, eventPayload } ) { + // Add server info + eventPayload.server = await getServerInfo() + eventPayload.server.canonicalUrl = process.env.CANONICAL_URL + delete eventPayload.server.id + + // Add stream info + if ( eventPayload.streamId ) { + eventPayload.stream = await getStream( { streamId: eventPayload.streamId, userId: eventPayload.userId } ) + } + + // Add user info (except email and pwd) + if ( eventPayload.userId ) { + eventPayload.user = await Users( ).where( { id: eventPayload.userId } ).select( '*' ).first( ) + if ( eventPayload.user ) { + delete eventPayload.user.passwordDigest + delete eventPayload.user.email + } + } + let { rows } = await knex.raw( ` SELECT * FROM webhooks_config WHERE "streamId" = ? `, [ streamId ] ) for ( let wh of rows ) { + if ( !wh.enabled ) + continue if ( !( event in wh.triggers ) ) continue + // Add webhook info (the key `webhook` will be replaced for each webhook configured, before serializing the payload and storing it) + eventPayload.webhook = wh + eventPayload.webhook.triggers = Object.keys( eventPayload.webhook.triggers ) + delete eventPayload.webhook.secret + await WebhooksEvents( ).insert( { id: crs( { length: 20 } ), webhookId: wh.id, - payload: eventPayload + payload: JSON.stringify( eventPayload ) } ) } }, diff --git a/packages/webhook-service/src/main.js b/packages/webhook-service/src/main.js index f059e1ac5..dc32c5e7b 100644 --- a/packages/webhook-service/src/main.js +++ b/packages/webhook-service/src/main.js @@ -28,11 +28,9 @@ async function doTask( task ) { let { rows } = await knex.raw( ` SELECT ev.payload as evt, - cnf.id as wh_id, cnf.url as wh_url, cnf.description as wh_desc, cnf.secret as wh_secret, cnf.enabled as wh_enabled, - stm.id as stm_id, stm.name as stm_name, stm.description as stm_desc, stm."isPublic" as stm_pub + cnf.id as wh_id, cnf.url as wh_url, cnf.secret as wh_secret, cnf.enabled as wh_enabled FROM webhooks_events ev INNER JOIN webhooks_config cnf ON ev."webhookId" = cnf.id - INNER JOIN streams stm ON cnf."streamId" = stm.id WHERE ev.id = ? LIMIT 1 `, [ task.id ] ) @@ -40,20 +38,15 @@ async function doTask( task ) { if ( !info ) { throw new Error( 'Internal error: DB inconsistent' ) } - if ( !info.wh_enabled ) return - let fullPayload = { - stream: { id: info.stm_id, name: info.stm_name, description: info.stm_desc, isPublic: info.stm_pub }, - webhook: { id: info.wh_id, url: info.wh_url, description: info.wh_desc }, - event: JSON.parse( info.evt ) - } + let fullPayload = JSON.parse( info.evt ) - let postData = { payload: JSON.stringify( fullPayload ) } + let postData = { payload: info.evt } let signature = crypto.createHmac( 'sha256', info.wh_secret || '' ).update( postData.payload ).digest( 'hex' ) let postHeaders = { 'X-WEBHOOK-SIGNATURE': signature } - console.log( `Callin webhook ${fullPayload.stream.id} : ${fullPayload.event.event_name} at ${fullPayload.webhook.url}...` ) + console.log( `Callin webhook ${fullPayload.streamId} : ${fullPayload.event.event_name} at ${fullPayload.webhook.url}...` ) let result = await makeNetworkRequest( { url: info.wh_url, data: postData, headersData: postHeaders } ) console.log( ` Result: ${JSON.stringify( result )}` ) From d629133e41fde75ba8916cea8c8fe6934a7c96fb Mon Sep 17 00:00:00 2001 From: cristi8 Date: Wed, 21 Jul 2021 21:02:53 +0300 Subject: [PATCH 21/21] fixed tests --- packages/server/modules/webhooks/tests/webhooks.spec.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/server/modules/webhooks/tests/webhooks.spec.js b/packages/server/modules/webhooks/tests/webhooks.spec.js index 21473dee1..e1bb34882 100644 --- a/packages/server/modules/webhooks/tests/webhooks.spec.js +++ b/packages/server/modules/webhooks/tests/webhooks.spec.js @@ -102,10 +102,10 @@ describe( 'Webhooks @webhooks', () => { } ) it( 'Should dispatch and get events', async () => { - await dispatchStreamEvent( { streamId: streamOne.id, event: 'commit_create', eventPayload: 'payload123' } ) + await dispatchStreamEvent( { streamId: streamOne.id, event: 'commit_create', eventPayload: { test: 'payload123' } } ) let lastEvents = await getLastWebhookEvents( { webhookId: webhookOne.id } ) expect( lastEvents ).to.have.lengthOf( 1 ) - expect( lastEvents[ 0 ].payload ).to.equal( 'payload123' ) + expect( JSON.parse( lastEvents[ 0 ].payload ).test ).to.equal( 'payload123' ) } ) } ) @@ -151,7 +151,7 @@ describe( 'Webhooks @webhooks', () => { } ) it( 'Should get stream webhooks and the previous events', async () => { - await dispatchStreamEvent( { streamId: streamTwo.id, event: 'commit_create', eventPayload: 'payload321' } ) + await dispatchStreamEvent( { streamId: streamTwo.id, event: 'commit_create', eventPayload: { test: 'payload321' } } ) const res = await sendRequest( userTwo.token, { query: `query { stream(id: "${streamTwo.id}") { webhooks { totalCount items { id url enabled @@ -165,7 +165,7 @@ describe( 'Webhooks @webhooks', () => { expect( webhooks.totalCount ).to.equal( 1 ) expect( webhooks.items[ 0 ].url ).to.equal( webhookTwo.url ) expect( webhooks.items[ 0 ].history.totalCount ).to.equal( 1 ) - expect( webhooks.items[ 0 ].history.items[ 0 ].payload ).to.equal( 'payload321' ) + expect( JSON.parse( webhooks.items[ 0 ].history.items[ 0 ].payload ).test ).to.equal( 'payload321' ) } ) it( 'Should update a webhook', async () => {