diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f9c3efed..72b6d5557 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,10 +1,22 @@ version: 2.1 +orbs: + snyk: snyk/snyk@1.4.0 + workflows: version: 2 test-build: jobs: + - vulnerability-scan: + context: &snyk-context + - snyk + filters: + branches: + only: + - main + - hotfix* + - test-server: filters: &filters-allow-all tags: @@ -14,14 +26,6 @@ workflows: - get-version: filters: *filters-allow-all - - build-approval: - type: approval - filters: &filters-ignore-main-branch-all-tags - branches: - ignore: main - tags: - ignore: /.*/ - - pre-commit: filters: *filters-allow-all @@ -32,51 +36,48 @@ workflows: requires: - test-server - get-version - - build-approval - docker-build-frontend: filters: *filters-build requires: - get-version - - build-approval - docker-build-webhooks: filters: *filters-build requires: - get-version - test-server - - build-approval - docker-build-file-imports: filters: *filters-build requires: - get-version - test-server - - build-approval - docker-build-previews: filters: *filters-build requires: - get-version - test-server - - build-approval - docker-build-test-container: filters: *filters-build requires: - get-version - test-server - - build-approval - docker-build-monitor-container: filters: *filters-build requires: - get-version - - build-approval - publish-approval: type: approval - filters: *filters-ignore-main-branch-all-tags + filters: &filters-ignore-main-branch-or-all-tags + branches: + ignore: main + tags: + ignore: /.*/ - docker-publish-server: context: &docker-hub-context @@ -326,6 +327,35 @@ jobs: # path: packages/server/coverage/lcov-report # destination: package/server/coverage + vulnerability-scan: + # snyk can undertake most types of scans through GitHub integration + # which does not require integration with the CI + # but it is not possible to scan npm/yarn package.json + # because it requires node_modules + # therefore this scanning has to be triggered via the cli + docker: &docker-image + - image: cimg/python:3.9.15-node + resource_class: small + working_directory: *work-dir + steps: + - checkout + - restore_cache: + name: Restore Yarn Package Cache + keys: + - yarn-packages-server-{{ checksum "yarn.lock" }} + - run: + name: Install Dependencies + command: yarn + - save_cache: + name: Save Yarn Package Cache + key: yarn-packages-server-{{ checksum "yarn.lock" }} + paths: + - .yarn/cache + - .yarn/unplugged + - snyk/scan: + additional-arguments: --yarn-workspaces --strict-out-of-sync=false + fail-on-issues: false + docker-build: &build-job docker: &docker-image - image: cimg/python:3.9.15-node @@ -336,6 +366,11 @@ jobs: - attach_workspace: at: /tmp/ci/workspace - run: cat workspace/env-vars >> $BASH_ENV + - run: + name: 'Check if should proceed' + command: | + [[ "${CIRCLE_BRANCH}" != "main" && -z "${CIRCLE_PULL_REQUEST}" ]] && echo "Should not build, stopping" && circleci-agent step halt && exit 1 + echo "proceeding" - setup_remote_docker: # a weird issue with yarn installing packages throwing EPERM errors # this fixes it diff --git a/packages/fileimport-service/obj/import_file.py b/packages/fileimport-service/obj/import_file.py index 853440538..a63aa4f80 100644 --- a/packages/fileimport-service/obj/import_file.py +++ b/packages/fileimport-service/obj/import_file.py @@ -1,53 +1,58 @@ - +import sys, os import json from specklepy.objects import Base from specklepy.objects.other import RenderMaterial -from specklepy.objects.geometry import Mesh, Point, Line +from specklepy.objects.geometry import Mesh from specklepy.transports.server import ServerTransport from specklepy.api.client import SpeckleClient -from specklepy.api.credentials import get_default_account from specklepy.api import operations - -import sys, os - from obj_file import ObjFile -TMP_RESULTS_PATH = '/tmp/import_result.json' -DEFAULT_BRANCH = 'uploads' + +TMP_RESULTS_PATH = "/tmp/import_result.json" +DEFAULT_BRANCH = "uploads" + def convert_material(obj_mat): speckle_mat = RenderMaterial() - speckle_mat.name = obj_mat['name'] - if 'diffuse' in obj_mat: - argb = [1,] + obj_mat['diffuse'] - speckle_mat.diffuse = int.from_bytes([int(val * 255) for val in argb], byteorder="big", signed=True) - if 'dissolved' in obj_mat: - speckle_mat.opacity = obj_mat['dissolved'] + speckle_mat.name = obj_mat["name"] + if "diffuse" in obj_mat: + argb = [ + 1, + ] + obj_mat["diffuse"] + speckle_mat.diffuse = int.from_bytes( + [int(val * 255) for val in argb], byteorder="big", signed=True + ) + if "dissolved" in obj_mat: + speckle_mat.opacity = obj_mat["dissolved"] return speckle_mat + def import_obj(): - file_path, user_id, stream_id, branch_name, commit_message = sys.argv[1:] - print(f'ImportOBJ argv[1:]: {sys.argv[1:]}') + file_path, _, stream_id, branch_name, commit_message = sys.argv[1:] + print(f"ImportOBJ argv[1:]: {sys.argv[1:]}") # Parse input obj = ObjFile(file_path) - print(f'Parsed obj with {len(obj.faces)} faces ({len(obj.vertices) * 3} vertices)') + print(f"Parsed obj with {len(obj.faces)} faces ({len(obj.vertices) * 3} vertices)") speckle_root = Base() - speckle_root['@objects'] = [] + speckle_root["@objects"] = [] for objname in obj.objects: - print(f' Converting {objname}...') + print(f" Converting {objname}...") speckle_obj = Base() speckle_obj.name = objname - speckle_obj['@displayValue'] = [] - speckle_root['@objects'].append(speckle_obj) - + speckle_obj["@displayValue"] = [] + speckle_root["@objects"].append(speckle_obj) + for obj_mesh in obj.objects[objname]: - speckle_vertices = [coord for point in obj_mesh['vertices'] for coord in point] + speckle_vertices = [ + coord for point in obj_mesh["vertices"] for coord in point + ] speckle_faces = [] - for obj_face in obj_mesh['faces']: + for obj_face in obj_mesh["faces"]: if len(obj_face) == 3: speckle_faces.append(0) elif len(obj_face) == 4: @@ -57,64 +62,75 @@ def import_obj(): speckle_faces.extend(obj_face) has_vertex_colors = False - for vc in obj_mesh['vertex_colors']: + for vc in obj_mesh["vertex_colors"]: if vc is not None: has_vertex_colors = True colors = [] if has_vertex_colors: - for vc in obj_mesh['vertex_colors']: + for vc in obj_mesh["vertex_colors"]: if vc is None: r, g, b = (1.0, 1.0, 1.0) else: r, g, b = vc argb = (1.0, r, g, b) - color = int.from_bytes([int(val * 255) for val in argb], byteorder="big", signed=True) + color = int.from_bytes( + [int(val * 255) for val in argb], byteorder="big", signed=True + ) colors.append(color) speckle_mesh = Mesh( vertices=speckle_vertices, faces=speckle_faces, colors=colors, - textureCoordinates=[] + textureCoordinates=[], ) - obj_material = obj_mesh['material'] + obj_material = obj_mesh["material"] if obj_material: - speckle_mesh['renderMaterial'] = convert_material(obj_material) + speckle_mesh["renderMaterial"] = convert_material(obj_material) - speckle_obj['@displayValue'].append(speckle_mesh) + speckle_obj["@displayValue"].append(speckle_mesh) # Commit - client = SpeckleClient(host=os.getenv('SPECKLE_SERVER_URL', 'localhost:3000'), use_ssl=False) - client.authenticate(os.environ['USER_TOKEN']) + client = SpeckleClient( + host=os.getenv("SPECKLE_SERVER_URL", "localhost:3000"), use_ssl=False + ) + client.authenticate_with_token(os.environ["USER_TOKEN"]) if not client.branch.get(stream_id, branch_name): - client.branch.create(stream_id, branch_name, 'File upload branch' if branch_name == 'uploads' else '') + client.branch.create( + stream_id, + branch_name, + "File upload branch" if branch_name == "uploads" else "", + ) transport = ServerTransport(client=client, stream_id=stream_id) - id = operations.send(base=speckle_root, transports=[transport], use_default_cache=False) + id = operations.send( + base=speckle_root, transports=[transport], use_default_cache=False + ) commit_id = client.commit.create( stream_id=stream_id, object_id=id, branch_name=(branch_name or DEFAULT_BRANCH), - message=(commit_message or 'OBJ file upload'), - source_application='OBJ' + message=(commit_message or "OBJ file upload"), + source_application="OBJ", ) return commit_id -if __name__ == '__main__': +if __name__ == "__main__": + from pathlib import Path + try: commit_id = import_obj() if not commit_id: raise Exception("Can't create commit") - results = {'success': True, 'commitId': commit_id} + results = {"success": True, "commitId": commit_id} except Exception as ex: - print('ERROR: ' + str(ex)) - results = {'success': False, 'error': str(ex)} + print("ERROR: " + str(ex)) + results = {"success": False, "error": str(ex)} - with open(TMP_RESULTS_PATH, 'w') as f: - json.dump(results, f) + Path(TMP_RESULTS_PATH).write_text(json.dumps(results)) diff --git a/packages/fileimport-service/requirements.txt b/packages/fileimport-service/requirements.txt index 7b3349251..f7e1614c8 100644 --- a/packages/fileimport-service/requirements.txt +++ b/packages/fileimport-service/requirements.txt @@ -1,2 +1,2 @@ numpy-stl==2.17.1 -specklepy==2.9.0 +specklepy==2.9.1 diff --git a/packages/fileimport-service/stl/import_file.py b/packages/fileimport-service/stl/import_file.py index 8de5c4f95..ac7d8740c 100644 --- a/packages/fileimport-service/stl/import_file.py +++ b/packages/fileimport-service/stl/import_file.py @@ -1,46 +1,50 @@ - import json import stl -from specklepy.objects.geometry import Mesh, Point +from specklepy.objects.geometry import Mesh from specklepy.transports.server import ServerTransport from specklepy.api.client import SpeckleClient -from specklepy.api.credentials import get_default_account from specklepy.api import operations import sys, os -TMP_RESULTS_PATH = '/tmp/import_result.json' -DEFAULT_BRANCH = 'uploads' +TMP_RESULTS_PATH = "/tmp/import_result.json" +DEFAULT_BRANCH = "uploads" + def import_stl(): - file_path, user_id, stream_id, branch_name, commit_message = sys.argv[1:] - print(f'ImportSTL argv[1:]: {sys.argv[1:]}') + file_path, _, stream_id, branch_name, commit_message = sys.argv[1:] + print(f"ImportSTL argv[1:]: {sys.argv[1:]}") # Parse input stl_mesh = stl.mesh.Mesh.from_file(file_path) - print(f'Parsed mesh with {stl_mesh.points.shape[0]} faces ({stl_mesh.points.shape[0] * 3} vertices)') + print( + f"Parsed mesh with {stl_mesh.points.shape[0]} faces ({stl_mesh.points.shape[0] * 3} vertices)" + ) # Construct speckle obj vertices = stl_mesh.points.flatten().tolist() faces = [] for i in range(stl_mesh.points.shape[0]): - faces.extend([0, 3*i, 3*i+1, 3*i+2]) + faces.extend([0, 3 * i, 3 * i + 1, 3 * i + 2]) speckle_mesh = Mesh( - vertices=vertices, - faces=faces, - colors=[], - textureCoordinates=[] + vertices=vertices, faces=faces, colors=[], textureCoordinates=[] ) - print('Constructed Speckle Mesh object') + print("Constructed Speckle Mesh object") # Commit - client = SpeckleClient(host=os.getenv('SPECKLE_SERVER_URL', 'localhost:3000'), use_ssl=False) - client.authenticate(os.environ['USER_TOKEN']) + client = SpeckleClient( + host=os.getenv("SPECKLE_SERVER_URL", "localhost:3000"), use_ssl=False + ) + client.authenticate_with_token(os.environ["USER_TOKEN"]) if not client.branch.get(stream_id, branch_name): - client.branch.create(stream_id, branch_name, 'File upload branch' if branch_name == 'uploads' else '') + client.branch.create( + stream_id, + branch_name, + "File upload branch" if branch_name == "uploads" else "", + ) transport = ServerTransport(client=client, stream_id=stream_id) id = operations.send( @@ -53,20 +57,22 @@ def import_stl(): stream_id=stream_id, object_id=id, branch_name=(branch_name or DEFAULT_BRANCH), - message=(commit_message or 'STL file upload'), - source_application='STL' + message=(commit_message or "STL file upload"), + source_application="STL", ) return commit_id -if __name__ == '__main__': +if __name__ == "__main__": + from pathlib import Path + try: commit_id = import_stl() - results = {'success': True, 'commitId': commit_id} + results = {"success": True, "commitId": commit_id} except Exception as ex: results = {'success': False, 'error': str(ex)} print(ex) - with open(TMP_RESULTS_PATH, 'w') as f: - json.dump(results, f) + print(results) + Path(TMP_RESULTS_PATH).write_text(json.dumps(results)) diff --git a/packages/preview-service/bg_service/prometheusMetrics.js b/packages/preview-service/bg_service/prometheusMetrics.js index 6de7db95d..4e3c84f73 100644 --- a/packages/preview-service/bg_service/prometheusMetrics.js +++ b/packages/preview-service/bg_service/prometheusMetrics.js @@ -100,7 +100,7 @@ module.exports = { metricDuration: new prometheusClient.Histogram({ name: 'speckle_server_operation_duration', help: 'Summary of the operation durations in seconds', - buckets: [0.5, 1, 5, 10, 30, 60, 300, 600], + buckets: [0.5, 1, 5, 10, 30, 60, 300, 600, 1200, 1800], labelNames: ['op'] }), diff --git a/packages/preview-service/routes/preview.js b/packages/preview-service/routes/preview.js index c55b9ffc7..8f9d03291 100644 --- a/packages/preview-service/routes/preview.js +++ b/packages/preview-service/routes/preview.js @@ -126,6 +126,12 @@ async function getScreenshot(objectUrl) { } router.get('/:streamId/:objectId', async function (req, res) { + const safeParamRgx = /^[\w]+$/i + const { streamId, objectId } = req.params || {} + if (!safeParamRgx.test(streamId) || !safeParamRgx.test(objectId)) { + return res.status(400).json({ error: 'Invalid streamId or objectId!' }) + } + const objectUrl = `http://127.0.0.1:3001/streams/${req.params.streamId}/objects/${req.params.objectId}` /* let authToken = '' diff --git a/packages/server/modules/activitystream/services/index.js b/packages/server/modules/activitystream/services/index.js index 76c47c62b..9b0901bc9 100644 --- a/packages/server/modules/activitystream/services/index.js +++ b/packages/server/modules/activitystream/services/index.js @@ -40,7 +40,7 @@ module.exports = { data: info } } - dispatchStreamEvent({ + await dispatchStreamEvent({ streamId, event: actionType, eventPayload: webhooksPayload diff --git a/packages/server/modules/auth/errors/index.ts b/packages/server/modules/auth/errors/index.ts new file mode 100644 index 000000000..89050725d --- /dev/null +++ b/packages/server/modules/auth/errors/index.ts @@ -0,0 +1,6 @@ +import { BaseError } from '@/modules/shared/errors' + +export class InvalidAccessCodeRequestError extends BaseError { + static code = 'INVALID_ACCESS_CODE_REQUEST' + static defaultMessage = 'An issue occurred while generating an access code for an app' +} diff --git a/packages/server/modules/auth/rest/index.js b/packages/server/modules/auth/rest/index.js index 302a70594..391beb280 100644 --- a/packages/server/modules/auth/rest/index.js +++ b/packages/server/modules/auth/rest/index.js @@ -13,6 +13,8 @@ const { const { validateToken, revokeTokenById } = require(`@/modules/core/services/tokens`) const { revokeRefreshToken } = require(`@/modules/auth/services/apps`) const { validateScopes } = require(`@/modules/shared`) +const { InvalidAccessCodeRequestError } = require('@/modules/auth/errors') +const { ForbiddenError } = require('apollo-server-errors') // TODO: Secure these endpoints! module.exports = (app) => { @@ -24,14 +26,17 @@ module.exports = (app) => { try { const appId = req.query.appId const app = await getApp({ id: appId }) - if (!app) throw new Error('App does not exist.') + + if (!app) throw new InvalidAccessCodeRequestError('App does not exist.') const challenge = req.query.challenge const userToken = req.query.token + if (!challenge) throw new InvalidAccessCodeRequestError('Missing challenge') + if (!userToken) throw new InvalidAccessCodeRequestError('Missing token') // 1. Validate token const { valid, scopes, userId } = await validateToken(userToken) - if (!valid) throw new Error('Invalid token') + if (!valid) throw new InvalidAccessCodeRequestError('Invalid token') // 2. Validate token scopes await validateScopes(scopes, 'tokens:write') @@ -41,7 +46,17 @@ module.exports = (app) => { } catch (err) { sentry({ err }) debug('speckle:error')(err) - return res.status(400).send(err.message) + + if ( + err instanceof InvalidAccessCodeRequestError || + err instanceof ForbiddenError + ) { + return res.status(400).send(err.message) + } else { + return res + .status(500) + .send('Something went wrong while processing your request') + } } }) diff --git a/packages/server/modules/auth/strategies.js b/packages/server/modules/auth/strategies.js index 618f91b02..38d4ddb9f 100644 --- a/packages/server/modules/auth/strategies.js +++ b/packages/server/modules/auth/strategies.js @@ -7,6 +7,7 @@ const passport = require('passport') const sentry = require('@/logging/sentryHelper') const { createAuthorizationCode } = require('./services/apps') +const { isSSLServer } = require('@/modules/shared/helpers/envHelper') /** * TODO: Get rid of session entirely, we don't use it for the app and it's not really necessary for the auth flow, so it only complicates things @@ -24,7 +25,10 @@ module.exports = async (app) => { secret: process.env.SESSION_SECRET, saveUninitialized: false, resave: false, - cookie: { maxAge: 1000 * 60 * 3 } // 3 minutes + cookie: { + maxAge: 1000 * 60 * 3, // 3 minutes + secure: isSSLServer() + } }) /** diff --git a/packages/server/modules/auth/strategies/local.js b/packages/server/modules/auth/strategies/local.js index 2d84acfde..80896560e 100644 --- a/packages/server/modules/auth/strategies/local.js +++ b/packages/server/modules/auth/strategies/local.js @@ -12,6 +12,7 @@ const { finalizeInvitedServerRegistration, resolveAuthRedirectPath } = require('@/modules/serverinvites/services/inviteProcessingService') +const { getIpFromRequest } = require('@/modules/shared/utils/ip') module.exports = async (app, session, sessionAppId, finalizeAuth) => { const strategy = { @@ -57,21 +58,8 @@ module.exports = async (app, session, sessionAppId, finalizeAuth) => { if (!req.body.password) throw new Error('Password missing') const user = req.body - user.ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress || '' - const ignorePrefixes = [ - '192.168.', - '10.', - '127.', - '172.1', - '172.2', - '172.3', - '::' - ] - for (const ipPrefix of ignorePrefixes) - if (user.ip.startsWith(ipPrefix)) { - delete user.ip - break - } + const ip = getIpFromRequest(req) + if (ip) user.ip = ip if ( user.ip && !(await respectsLimits({ action: 'USER_CREATE', source: user.ip })) diff --git a/packages/server/modules/blobstorage/index.js b/packages/server/modules/blobstorage/index.js index f5df6e8b4..965fff74b 100644 --- a/packages/server/modules/blobstorage/index.js +++ b/packages/server/modules/blobstorage/index.js @@ -31,6 +31,8 @@ const { getFileSizeLimit } = require('@/modules/blobstorage/services') +const { isArray } = require('lodash') + const { NotFoundError, ResourceMismatch, @@ -176,6 +178,12 @@ exports.init = async (app) => { allowAnonymousUsersOnPublicStreams ]), async (req, res) => { + if (!isArray(req.body)) { + return res + .status(400) + .json({ error: 'An array of blob IDs expected in the body.' }) + } + const bq = await getAllStreamBlobIds({ streamId: req.params.streamId }) const unknownBlobIds = req.body.filter( (id) => bq.findIndex((bInfo) => bInfo.id === id) === -1 diff --git a/packages/server/modules/core/rest/diffDownload.js b/packages/server/modules/core/rest/diffDownload.js index 7f9387d59..f6fd1b0e0 100644 --- a/packages/server/modules/core/rest/diffDownload.js +++ b/packages/server/modules/core/rest/diffDownload.js @@ -7,6 +7,9 @@ const { contextMiddleware } = require('@/modules/shared') const { validatePermissionsReadStream } = require('./authUtils') const { SpeckleObjectsStream } = require('./speckleObjectsStream') const { getObjectsStream } = require('../services/objects') +const { + rejectsRequestWithRatelimitStatusIfNeeded +} = require('@/modules/core/services/ratelimits') const { pipeline, PassThrough } = require('stream') @@ -14,6 +17,12 @@ module.exports = (app) => { app.options('/api/getobjects/:streamId', cors()) app.post('/api/getobjects/:streamId', cors(), contextMiddleware, async (req, res) => { + const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({ + action: 'POST /api/getobjects/:streamId', + req, + res + }) + if (rejected) return rejected const hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req diff --git a/packages/server/modules/core/rest/diffUpload.js b/packages/server/modules/core/rest/diffUpload.js index 38df741e0..fb33d8fe9 100644 --- a/packages/server/modules/core/rest/diffUpload.js +++ b/packages/server/modules/core/rest/diffUpload.js @@ -5,6 +5,9 @@ const debug = require('debug') const { contextMiddleware } = require('@/modules/shared') const { validatePermissionsWriteStream } = require('./authUtils') +const { + rejectsRequestWithRatelimitStatusIfNeeded +} = require('@/modules/core/services/ratelimits') const { hasObjects } = require('../services/objects') @@ -12,6 +15,12 @@ module.exports = (app) => { app.options('/api/diff/:streamId', cors()) app.post('/api/diff/:streamId', cors(), contextMiddleware, async (req, res) => { + const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({ + action: 'POST /api/diff/:streamId', + req, + res + }) + if (rejected) return rejected const hasStreamAccess = await validatePermissionsWriteStream( req.params.streamId, req diff --git a/packages/server/modules/core/rest/download.js b/packages/server/modules/core/rest/download.js index af71d0eb2..644ffea62 100644 --- a/packages/server/modules/core/rest/download.js +++ b/packages/server/modules/core/rest/download.js @@ -9,6 +9,9 @@ const { validatePermissionsReadStream } = require('./authUtils') const { getObject, getObjectChildrenStream } = require('../services/objects') const { SpeckleObjectsStream } = require('./speckleObjectsStream') const { pipeline, PassThrough } = require('stream') +const { + rejectsRequestWithRatelimitStatusIfNeeded +} = require('@/modules/core/services/ratelimits') module.exports = (app) => { app.options('/objects/:streamId/:objectId', cors()) @@ -18,6 +21,13 @@ module.exports = (app) => { cors(), contextMiddleware, async (req, res) => { + const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({ + action: 'GET /objects/:streamId/:objectId', + req, + res + }) + if (rejected) return rejected + const hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req @@ -85,6 +95,13 @@ module.exports = (app) => { cors(), contextMiddleware, async (req, res) => { + const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({ + action: 'GET /objects/:streamId/:objectId/single', + req, + res + }) + if (rejected) return rejected + const hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req diff --git a/packages/server/modules/core/rest/upload.js b/packages/server/modules/core/rest/upload.js index 2448c340e..68ca0dd50 100644 --- a/packages/server/modules/core/rest/upload.js +++ b/packages/server/modules/core/rest/upload.js @@ -8,6 +8,9 @@ const { contextMiddleware } = require('@/modules/shared') const { validatePermissionsWriteStream } = require('./authUtils') const { createObjectsBatched } = require('../services/objects') +const { + rejectsRequestWithRatelimitStatusIfNeeded +} = require('@/modules/core/services/ratelimits') const MAX_FILE_SIZE = 50 * 1024 * 1024 @@ -15,6 +18,13 @@ module.exports = (app) => { app.options('/objects/:streamId', cors()) app.post('/objects/:streamId', cors(), contextMiddleware, async (req, res) => { + const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({ + action: 'POST /objects/:streamId', + req, + res + }) + if (rejected) return rejected + const hasStreamAccess = await validatePermissionsWriteStream( req.params.streamId, req diff --git a/packages/server/modules/core/services/ratelimits.js b/packages/server/modules/core/services/ratelimits.js index 7b5043bd0..d5cad08ed 100644 --- a/packages/server/modules/core/services/ratelimits.js +++ b/packages/server/modules/core/services/ratelimits.js @@ -25,7 +25,13 @@ const LIMITS = { BRANCHES: parseInt(process.env.LIMIT_BRANCHES) || 1000, // per stream TOKENS: parseInt(process.env.LIMIT_TOKENS) || 1000, // per user ACTIVE_SUBSCRIPTIONS: parseInt(process.env.LIMIT_ACTIVE_SUBSCRIPTIONS) || 100, // per user - ACTIVE_CONNECTIONS: parseInt(process.env.LIMIT_ACTIVE_CONNECTIONS) || 100 // per source ip + ACTIVE_CONNECTIONS: parseInt(process.env.LIMIT_ACTIVE_CONNECTIONS) || 100, // per source ip + + 'POST /api/getobjects/:streamId': 200, // for 1 minute + 'POST /api/diff/:streamId': 200, // for 1 minute + 'POST /objects/:streamId': 200, // for 1 minute + 'GET /objects/:streamId/:objectId': 200, // for 1 minute + 'GET /objects/:streamId/:objectId/single': 200 // for 1 minute } const LIMIT_INTERVAL = { @@ -42,7 +48,13 @@ const LIMIT_INTERVAL = { BRANCHES: 0, TOKENS: 0, ACTIVE_SUBSCRIPTIONS: 0, - ACTIVE_CONNECTIONS: 0 + ACTIVE_CONNECTIONS: 0, + + 'POST /api/getobjects/:streamId': 60, + 'POST /api/diff/:streamId': 60, + 'POST /objects/:streamId': 60, + 'GET /objects/:streamId/:objectId': 60, + 'GET /objects/:streamId/:objectId/single': 60 } const rateLimitedCache = {} @@ -74,22 +86,31 @@ async function shouldRateLimitNext({ action, source }) { return shouldRateLimit } +// returns true if the action is fine, false if it should be blocked because of exceeding limit +async function respectsLimits({ action, source }) { + const rateLimitKey = `${action} ${source}` + const promise = shouldRateLimitNext({ action, source }).then((shouldRateLimit) => { + if (shouldRateLimit) rateLimitedCache[rateLimitKey] = true + else delete rateLimitedCache[rateLimitKey] + }) + if (rateLimitedCache[rateLimitKey]) { + await promise + } + + if (rateLimitedCache[rateLimitKey]) limitsReached.labels(action).inc() + return !rateLimitedCache[rateLimitKey] +} + +async function rejectsRequestWithRatelimitStatusIfNeeded({ action, req, res }) { + const source = req.context.userId || req.context.ip + if (!(await respectsLimits({ action, source }))) + return res.status(429).set('X-Speckle-Meditation', 'https://http.cat/429').send({ + err: 'You are sending too many requests. You have been rate limited. Please try again later.' + }) +} module.exports = { LIMITS, LIMIT_INTERVAL, - - // returns true if the action is fine, false if it should be blocked because of exceeding limit - async respectsLimits({ action, source }) { - const rateLimitKey = `${action} ${source}` - const promise = shouldRateLimitNext({ action, source }).then((shouldRateLimit) => { - if (shouldRateLimit) rateLimitedCache[rateLimitKey] = true - else delete rateLimitedCache[rateLimitKey] - }) - if (rateLimitedCache[rateLimitKey]) { - await promise - } - - if (rateLimitedCache[rateLimitKey]) limitsReached.labels(action).inc() - return !rateLimitedCache[rateLimitKey] - } + respectsLimits, + rejectsRequestWithRatelimitStatusIfNeeded } diff --git a/packages/server/modules/index.js b/packages/server/modules/index.js index bedfdb12a..a3b617b41 100644 --- a/packages/server/modules/index.js +++ b/packages/server/modules/index.js @@ -55,7 +55,8 @@ async function getSpeckleModules() { './blobstorage', './notifications', './activitystream', - './accessrequests' + './accessrequests', + './webhooks' ] for (const dir of moduleDirs) { diff --git a/packages/server/modules/shared/helpers/envHelper.ts b/packages/server/modules/shared/helpers/envHelper.ts index 3a6626c3e..fba084f22 100644 --- a/packages/server/modules/shared/helpers/envHelper.ts +++ b/packages/server/modules/shared/helpers/envHelper.ts @@ -56,3 +56,10 @@ export function shouldDisableNotificationsConsumption() { process.env.DISABLE_NOTIFICATIONS_CONSUMPTION || 'false' ) } + +/** + * Check whether we're running an SSL server + */ +export function isSSLServer() { + return /^https:\/\//.test(getBaseUrl()) +} diff --git a/packages/server/modules/shared/index.js b/packages/server/modules/shared/index.js index 0af24a2bd..0923471d7 100644 --- a/packages/server/modules/shared/index.js +++ b/packages/server/modules/shared/index.js @@ -5,6 +5,7 @@ const { ForbiddenError, ApolloError } = require('apollo-server-express') const { RedisPubSub } = require('graphql-redis-subscriptions') const { buildRequestLoaders } = require('@/modules/core/loaders') const { validateToken } = require(`@/modules/core/services/tokens`) +const { getIpFromRequest } = require('@/modules/shared/utils/ip') const StreamPubsubEvents = Object.freeze({ UserStreamAdded: 'USER_STREAM_ADDED', @@ -49,7 +50,7 @@ function addLoadersToCtx(ctx) { async function buildContext({ req, connection }) { // Parsing auth info const ctx = await contextApiTokenHelper({ req, connection }) - + ctx.ip = getIpFromRequest(req) // Adding request data loaders return addLoadersToCtx(ctx) } diff --git a/packages/server/modules/shared/utils/ip.js b/packages/server/modules/shared/utils/ip.js new file mode 100644 index 000000000..f8db74a66 --- /dev/null +++ b/packages/server/modules/shared/utils/ip.js @@ -0,0 +1,15 @@ +const getIpFromRequest = (req) => { + let ip + try { + ip = req.headers['cf-connecting-ip'] || req.ip || req.connection.remoteAddress || '' + } catch { + ip = '' + } + const ignorePrefixes = ['192.168.', '10.', '127.', '172.1', '172.2', '172.3', '::'] + + for (const ipPrefix of ignorePrefixes) + if (ip.startsWith(ipPrefix) || ip === '') return null + return ip +} + +module.exports = { getIpFromRequest } diff --git a/packages/server/modules/webhooks/index.js b/packages/server/modules/webhooks/index.js deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/server/modules/webhooks/index.ts b/packages/server/modules/webhooks/index.ts new file mode 100644 index 000000000..396eda435 --- /dev/null +++ b/packages/server/modules/webhooks/index.ts @@ -0,0 +1,27 @@ +import cron from 'node-cron' +import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' +import { modulesDebug } from '@/modules/shared/utils/logger' +import { scheduleExecution } from '@/modules/core/services/taskScheduler' +import { cleanOrphanedWebhookConfigs } from '@/modules/webhooks/services/cleanup' + +const webhooksDebug = modulesDebug.extend('activities') + +const scheduleWebhookCleanup = () => { + const cronExpression = '0 4 * * 1' + return scheduleExecution(cronExpression, 'weeklyWebhookCleanup', async () => { + webhooksDebug('Starting weekly webhooks cleanup') + await cleanOrphanedWebhookConfigs() + webhooksDebug('Finished cleanup') + }) +} + +let scheduledTask: cron.ScheduledTask | null = null + +export const init: SpeckleModule['init'] = () => { + modulesDebug('🎣 Init webhooks module') + scheduledTask = scheduleWebhookCleanup() +} + +export const shutdown: SpeckleModule['shutdown'] = () => { + if (scheduledTask) scheduledTask.stop() +} diff --git a/packages/server/modules/webhooks/migrations/20221104104921_webhooks_drop_stream_fk.ts b/packages/server/modules/webhooks/migrations/20221104104921_webhooks_drop_stream_fk.ts new file mode 100644 index 000000000..4597f2823 --- /dev/null +++ b/packages/server/modules/webhooks/migrations/20221104104921_webhooks_drop_stream_fk.ts @@ -0,0 +1,16 @@ +import { Knex } from 'knex' + +const TABLE_NAME = 'webhooks_config' + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable(TABLE_NAME, (table) => { + // dropping the foreign key and not adding a new one + // if we still had the constraint, the late triggered event ie stream_delete hooks + // would not find the webhook configs + table.dropForeign('streamId') + }) +} + +export async function down() { + return +} diff --git a/packages/server/modules/webhooks/services/cleanup.ts b/packages/server/modules/webhooks/services/cleanup.ts new file mode 100644 index 000000000..125c8244b --- /dev/null +++ b/packages/server/modules/webhooks/services/cleanup.ts @@ -0,0 +1,15 @@ +import knex from '@/db/knex' + +export async function cleanOrphanedWebhookConfigs() { + await knex.raw( + // i know im using a where in here, but this is used as background operation + ` + delete from webhooks_config + where id in ( + select wh.id from webhooks_config wh + left join streams st on wh."streamId" = st.id + where st.id is null + ) + ` + ) +} diff --git a/packages/server/modules/webhooks/services/webhooks.js b/packages/server/modules/webhooks/services/webhooks.js index 5a82387d5..07df06fd7 100644 --- a/packages/server/modules/webhooks/services/webhooks.js +++ b/packages/server/modules/webhooks/services/webhooks.js @@ -107,6 +107,8 @@ module.exports = { } } + // with this select, we must have the streamid available on the webhook config, + // even when the stream is deleted, to dispatch the stream deleted webhook events const { rows } = await knex.raw( ` SELECT * FROM webhooks_config WHERE "streamId" = ? diff --git a/packages/server/modules/webhooks/tests/cleanup.spec.ts b/packages/server/modules/webhooks/tests/cleanup.spec.ts new file mode 100644 index 000000000..94bde26c3 --- /dev/null +++ b/packages/server/modules/webhooks/tests/cleanup.spec.ts @@ -0,0 +1,81 @@ +import knex from '@/db/knex' +import { createStream } from '@/modules/core/services/streams' +import { createUser } from '@/modules/core/services/users' +import { cleanOrphanedWebhookConfigs } from '@/modules/webhooks/services/cleanup' +import { truncateTables } from '@/test/hooks' +import { expect } from 'chai' +import crs from 'crypto-random-string' + +const WEBHOOKS_CONFIG_TABLE = 'webhooks_config' +const WEBHOOKS_EVENTS_TABLE = 'webhooks_events' + +const WebhooksConfig = () => knex(WEBHOOKS_CONFIG_TABLE) +const randomId = () => crs({ length: 10 }) + +const countWebhooks = async () => { + const [{ count }] = await WebhooksConfig().count() + return parseInt(count as string) +} + +describe('Webhooks cleanup @webhooks', () => { + before(async () => { + await truncateTables([WEBHOOKS_CONFIG_TABLE, WEBHOOKS_EVENTS_TABLE]) + }) + + it('Cleans orphaned webhook configs', async () => { + const webhookConfig = { + id: randomId(), + streamId: randomId(), + url: 'foobar', + description: 'test_hook', + triggers: { + // eslint-disable-next-line camelcase + stream_update: true + } + } + await WebhooksConfig().insert(webhookConfig) + expect(await countWebhooks()).to.equal(1) + await cleanOrphanedWebhookConfigs() + expect(await countWebhooks()).to.equal(0) + }) + + it('Cleans orphans, leaves live ones intact', async () => { + const ownerId = await createUser({ + name: 'User', + email: 'user@gmail.com', + password: 'jdsadjsadasfdsa' + }) + const streamId = await createStream({ + name: 'foo', + description: 'bar', + ownerId + }) + + const webhookConfigs = [ + { + id: randomId(), + streamId: randomId(), + url: 'foobar', + description: 'test_hook', + triggers: { + // eslint-disable-next-line camelcase + stream_update: true + } + }, + { + id: randomId(), + streamId, + url: 'foobar', + description: 'test_hook', + triggers: { + // eslint-disable-next-line camelcase + stream_update: true + } + } + ] + await Promise.all(webhookConfigs.map((c) => WebhooksConfig().insert(c))) + expect(await countWebhooks()).to.equal(2) + await cleanOrphanedWebhookConfigs() + expect(await countWebhooks()).to.equal(1) + }) +}) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index a8cbc0795..009a47e28 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -503,27 +503,27 @@ export default class Sandbox { this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams this.viewer.requestRender() }) - staticAoFolder - .addInput(Sandbox.pipelineParams.staticAoParams, 'minDistance', { - min: 0, - max: 100, - step: 0.000001 - }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams - this.viewer.requestRender() - }) + // staticAoFolder + // .addInput(Sandbox.pipelineParams.staticAoParams, 'minDistance', { + // min: 0, + // max: 100, + // step: 0.000001 + // }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + // this.viewer.requestRender() + // }) - staticAoFolder - .addInput(Sandbox.pipelineParams.staticAoParams, 'maxDistance', { - min: 0, - max: 100, - step: 0.000001 - }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams - this.viewer.requestRender() - }) + // staticAoFolder + // .addInput(Sandbox.pipelineParams.staticAoParams, 'maxDistance', { + // min: 0, + // max: 100, + // step: 0.000001 + // }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + // this.viewer.requestRender() + // }) staticAoFolder .addInput(Sandbox.pipelineParams.staticAoParams, 'kernelRadius', { min: 0, diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 4cc5a026d..52b66ba6e 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -129,4 +129,10 @@ await sandbox.loadUrl( // 'https://speckle.xyz/streams/b85d53c3b4/commits/b47f21b707' // Crankshaft // 'https://speckle.xyz/streams/c239718aff/commits/b3a8cfb97d' + // Building AO params + // 'https://latest.speckle.dev/streams/0dd74866d0/commits/317e210afa' + // Murder Cube + // 'https://latest.speckle.dev/streams/c1faab5c62/commits/7f0c4d2fc1/' + // Classroom + // 'https://speckle.xyz/streams/0208ffb67b/commits/a980292728' ) diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index 83dba5c99..09121fb8a 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -264,6 +264,8 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` mat3 kernelMatrix = mat3( tangent, bitangent, viewNormal ); float occlusion = 0.0; float kernelSize_ws = computeKernelSize(-viewPosition.z, kernelRadius); + float div = float( KERNEL_SIZE); + float maxDist = kernelSize_ws / (cameraFar - cameraNear); for ( int i = 0; i < KERNEL_SIZE; i ++ ) { vec3 sampleVector = kernelMatrix * kernel[ i ]; // reorient sample vector in view space vec3 samplePoint = viewPosition + ( sampleVector * kernelSize_ws ); // calculate sample point @@ -273,11 +275,11 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` float realDepth = getLinearDepth( samplePointUv ); // get linear depth from depth texture float sampleDepth = viewZToOrthographicDepth( samplePoint.z + bias, cameraNear, cameraFar ); // compute linear depth of the sample view Z value float delta = sampleDepth - realDepth; - if ( delta > minDistance && delta < maxDistance ) { // if fragment is before sample point, increase occlusion + if ( delta > 0. && delta < maxDist ) { // if fragment is before sample point, increase occlusion occlusion += 1.0; } } - return clamp( occlusion * intensity / float( KERNEL_SIZE ), 0.0, 1.0 ); + return clamp( occlusion * intensity / div, 0.0, 1.0 ); #endif } void main() { diff --git a/utils/helm/speckle-server/templates/server/deployment.yml b/utils/helm/speckle-server/templates/server/deployment.yml index ab3e3defd..f34f5e465 100644 --- a/utils/helm/speckle-server/templates/server/deployment.yml +++ b/utils/helm/speckle-server/templates/server/deployment.yml @@ -70,7 +70,7 @@ spec: command: - node - -e - - require('request')('http://localhost:3000/graphql?query={serverInfo{version}}', (e,r,b) => process.exit(b.toLowerCase().includes('error'))) + - "require('request')({headers: {'Content-Type': 'application/json'}, uri: 'http://localhost:3000/graphql?query={serverInfo{version}}', method: 'GET' }, (e,r,b) => process.exit(r.statusCode != 200 || b.toLowerCase().includes('error')))" readinessProbe: initialDelaySeconds: 5 @@ -80,7 +80,7 @@ spec: command: - node - -e - - require('request')('http://localhost:3000/graphql?query={serverInfo{version}}', (e,r,b) => process.exit(b.toLowerCase().includes('error'))) + - "require('request')({headers: {'Content-Type': 'application/json'}, uri: 'http://localhost:3000/graphql?query={serverInfo{version}}', method: 'GET' }, (e,r,b) => process.exit(r.statusCode != 200 || b.toLowerCase().includes('error')))" env: - name: CANONICAL_URL diff --git a/utils/helm/speckle-server/templates/tests/deployment.yml b/utils/helm/speckle-server/templates/tests/deployment.yml index e29205717..25b977c26 100644 --- a/utils/helm/speckle-server/templates/tests/deployment.yml +++ b/utils/helm/speckle-server/templates/tests/deployment.yml @@ -1,49 +1,53 @@ {{- if .Values.helm_test_enabled }} -apiVersion: v1 -kind: Pod + +apiVersion: batch/v1 +kind: Job metadata: - name: "speckle-test-deployment" + name: "speckle-test" namespace: {{ .Values.namespace }} annotations: - "helm.sh/hook": test + helm.sh/hook: test labels: {{ include "test.labels" . | indent 4 }} spec: - containers: - - name: test-deployment - image: speckle/speckle-test-deployment:{{ .Values.docker_image_tag }} - env: - - name: SPECKLE_SERVER - value: https://{{ .Values.domain }} - - name: SERVER_VERSION - value: {{ .Values.docker_image_tag }} - resources: - requests: - cpu: {{ .Values.test.requests.cpu }} - memory: {{ .Values.test.requests.memory }} - limits: - cpu: {{ .Values.test.limits.cpu }} - memory: {{ .Values.test.limits.memory }} + backoffLimit: 1 + parallelism: 1 + completions: 1 + template: + spec: + containers: + - name: test-deployment + image: speckle/speckle-test-deployment:{{ .Values.docker_image_tag }} + env: + - name: SPECKLE_SERVER + value: https://{{ .Values.domain }} + - name: SERVER_VERSION + value: {{ .Values.docker_image_tag }} + resources: + requests: + cpu: {{ .Values.test.requests.cpu }} + memory: {{ .Values.test.requests.memory }} + limits: + cpu: {{ .Values.test.limits.cpu }} + memory: {{ .Values.test.limits.memory }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 20000 + restartPolicy: Never securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - privileged: false - readOnlyRootFilesystem: true runAsNonRoot: true runAsUser: 20000 + runAsGroup: 30000 + seccompProfile: + type: RuntimeDefault - restartPolicy: Never - - securityContext: - runAsNonRoot: true - runAsUser: 20000 - runAsGroup: 30000 - seccompProfile: - type: RuntimeDefault - - {{- if .Values.test.serviceAccount.create }} - serviceAccountName: {{ include "test.name" $ }} - {{- end }} + {{- if .Values.test.serviceAccount.create }} + serviceAccountName: {{ include "test.name" $ }} + {{- end }} {{- end }}