huanld/speckle-server
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Fixed conflicts

Browse Source
This commit is contained in:
AlexandruPopovici
2022-11-15 12:41:04 +02:00
parent c654736c50 f0524b53db
commit 543fb765af
32 changed files with 511 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+51 -16
View File
@@ -1,10 +1,22 @@
version: 2.1
orbs:
snyk: snyk/snyk@1.4.0
workflows:
version: 2
test-build:
jobs:
- vulnerability-scan:
context: &snyk-context
- snyk
filters:
branches:
only:
- main
- hotfix*
- test-server:
filters: &filters-allow-all
tags:
@@ -14,14 +26,6 @@ workflows:
- get-version:
filters: *filters-allow-all
- build-approval:
type: approval
filters: &filters-ignore-main-branch-all-tags
branches:
ignore: main
tags:
ignore: /.*/
- pre-commit:
filters: *filters-allow-all
@@ -32,51 +36,48 @@ workflows:
requires:
- test-server
- get-version
- build-approval
- docker-build-frontend:
filters: *filters-build
requires:
- get-version
- build-approval
- docker-build-webhooks:
filters: *filters-build
requires:
- get-version
- test-server
- build-approval
- docker-build-file-imports:
filters: *filters-build
requires:
- get-version
- test-server
- build-approval
- docker-build-previews:
filters: *filters-build
requires:
- get-version
- test-server
- build-approval
- docker-build-test-container:
filters: *filters-build
requires:
- get-version
- test-server
- build-approval
- docker-build-monitor-container:
filters: *filters-build
requires:
- get-version
- build-approval
- publish-approval:
type: approval
filters: *filters-ignore-main-branch-all-tags
filters: &filters-ignore-main-branch-or-all-tags
branches:
ignore: main
tags:
ignore: /.*/
- docker-publish-server:
context: &docker-hub-context
@@ -326,6 +327,35 @@ jobs:
# path: packages/server/coverage/lcov-report
# destination: package/server/coverage
vulnerability-scan:
# snyk can undertake most types of scans through GitHub integration
# which does not require integration with the CI
# but it is not possible to scan npm/yarn package.json
# because it requires node_modules
# therefore this scanning has to be triggered via the cli
docker: &docker-image
- image: cimg/python:3.9.15-node
resource_class: small
working_directory: *work-dir
steps:
- checkout
- restore_cache:
name: Restore Yarn Package Cache
keys:
- yarn-packages-server-{{ checksum "yarn.lock" }}
- run:
name: Install Dependencies
command: yarn
- save_cache:
name: Save Yarn Package Cache
key: yarn-packages-server-{{ checksum "yarn.lock" }}
paths:
- .yarn/cache
- .yarn/unplugged
- snyk/scan:
additional-arguments: --yarn-workspaces --strict-out-of-sync=false
fail-on-issues: false
docker-build: &build-job
docker: &docker-image
- image: cimg/python:3.9.15-node
@@ -336,6 +366,11 @@ jobs:
- attach_workspace:
at: /tmp/ci/workspace
- run: cat workspace/env-vars >> $BASH_ENV
- run:
name: 'Check if should proceed'
command: |
[[ "${CIRCLE_BRANCH}" != "main" && -z "${CIRCLE_PULL_REQUEST}" ]] && echo "Should not build, stopping" && circleci-agent step halt && exit 1
echo "proceeding"
- setup_remote_docker:
# a weird issue with yarn installing packages throwing EPERM errors
# this fixes it
packages/fileimport-service/obj/import_file.py
+59 -43
View File
@@ -1,53 +1,58 @@
import sys, os
import json
from specklepy.objects import Base
from specklepy.objects.other import RenderMaterial
from specklepy.objects.geometry import Mesh, Point, Line
from specklepy.objects.geometry import Mesh
from specklepy.transports.server import ServerTransport
from specklepy.api.client import SpeckleClient
from specklepy.api.credentials import get_default_account
from specklepy.api import operations
import sys, os
from obj_file import ObjFile
TMP_RESULTS_PATH = '/tmp/import_result.json'
DEFAULT_BRANCH = 'uploads'
TMP_RESULTS_PATH = "/tmp/import_result.json"
DEFAULT_BRANCH = "uploads"
def convert_material(obj_mat):
speckle_mat = RenderMaterial()
speckle_mat.name = obj_mat['name']
if 'diffuse' in obj_mat:
argb = [1,] + obj_mat['diffuse']
speckle_mat.diffuse = int.from_bytes([int(val * 255) for val in argb], byteorder="big", signed=True)
if 'dissolved' in obj_mat:
speckle_mat.opacity = obj_mat['dissolved']
speckle_mat.name = obj_mat["name"]
if "diffuse" in obj_mat:
argb = [
1,
] + obj_mat["diffuse"]
speckle_mat.diffuse = int.from_bytes(
[int(val * 255) for val in argb], byteorder="big", signed=True
)
if "dissolved" in obj_mat:
speckle_mat.opacity = obj_mat["dissolved"]
return speckle_mat
def import_obj():
file_path, user_id, stream_id, branch_name, commit_message = sys.argv[1:]
print(f'ImportOBJ argv[1:]: {sys.argv[1:]}')
file_path, _, stream_id, branch_name, commit_message = sys.argv[1:]
print(f"ImportOBJ argv[1:]: {sys.argv[1:]}")
# Parse input
obj = ObjFile(file_path)
print(f'Parsed obj with {len(obj.faces)} faces ({len(obj.vertices) * 3} vertices)')
print(f"Parsed obj with {len(obj.faces)} faces ({len(obj.vertices) * 3} vertices)")
speckle_root = Base()
speckle_root['@objects'] = []
speckle_root["@objects"] = []
for objname in obj.objects:
print(f' Converting {objname}...')
print(f" Converting {objname}...")
speckle_obj = Base()
speckle_obj.name = objname
speckle_obj['@displayValue'] = []
speckle_root['@objects'].append(speckle_obj)
speckle_obj["@displayValue"] = []
speckle_root["@objects"].append(speckle_obj)
for obj_mesh in obj.objects[objname]:
speckle_vertices = [coord for point in obj_mesh['vertices'] for coord in point]
speckle_vertices = [
coord for point in obj_mesh["vertices"] for coord in point
]
speckle_faces = []
for obj_face in obj_mesh['faces']:
for obj_face in obj_mesh["faces"]:
if len(obj_face) == 3:
speckle_faces.append(0)
elif len(obj_face) == 4:
@@ -57,64 +62,75 @@ def import_obj():
speckle_faces.extend(obj_face)
has_vertex_colors = False
for vc in obj_mesh['vertex_colors']:
for vc in obj_mesh["vertex_colors"]:
if vc is not None:
has_vertex_colors = True
colors = []
if has_vertex_colors:
for vc in obj_mesh['vertex_colors']:
for vc in obj_mesh["vertex_colors"]:
if vc is None:
r, g, b = (1.0, 1.0, 1.0)
else:
r, g, b = vc
argb = (1.0, r, g, b)
color = int.from_bytes([int(val * 255) for val in argb], byteorder="big", signed=True)
color = int.from_bytes(
[int(val * 255) for val in argb], byteorder="big", signed=True
)
colors.append(color)
speckle_mesh = Mesh(
vertices=speckle_vertices,
faces=speckle_faces,
colors=colors,
textureCoordinates=[]
textureCoordinates=[],
)
obj_material = obj_mesh['material']
obj_material = obj_mesh["material"]
if obj_material:
speckle_mesh['renderMaterial'] = convert_material(obj_material)
speckle_mesh["renderMaterial"] = convert_material(obj_material)
speckle_obj['@displayValue'].append(speckle_mesh)
speckle_obj["@displayValue"].append(speckle_mesh)
# Commit
client = SpeckleClient(host=os.getenv('SPECKLE_SERVER_URL', 'localhost:3000'), use_ssl=False)
client.authenticate(os.environ['USER_TOKEN'])
client = SpeckleClient(
host=os.getenv("SPECKLE_SERVER_URL", "localhost:3000"), use_ssl=False
)
client.authenticate_with_token(os.environ["USER_TOKEN"])
if not client.branch.get(stream_id, branch_name):
client.branch.create(stream_id, branch_name, 'File upload branch' if branch_name == 'uploads' else '')
client.branch.create(
stream_id,
branch_name,
"File upload branch" if branch_name == "uploads" else "",
)
transport = ServerTransport(client=client, stream_id=stream_id)
id = operations.send(base=speckle_root, transports=[transport], use_default_cache=False)
id = operations.send(
base=speckle_root, transports=[transport], use_default_cache=False
)
commit_id = client.commit.create(
stream_id=stream_id,
object_id=id,
branch_name=(branch_name or DEFAULT_BRANCH),
message=(commit_message or 'OBJ file upload'),
source_application='OBJ'
message=(commit_message or "OBJ file upload"),
source_application="OBJ",
)
return commit_id
if __name__ == '__main__':
if __name__ == "__main__":
from pathlib import Path
try:
commit_id = import_obj()
if not commit_id:
raise Exception("Can't create commit")
results = {'success': True, 'commitId': commit_id}
results = {"success": True, "commitId": commit_id}
except Exception as ex:
print('ERROR: ' + str(ex))
results = {'success': False, 'error': str(ex)}
print("ERROR: " + str(ex))
results = {"success": False, "error": str(ex)}
with open(TMP_RESULTS_PATH, 'w') as f:
json.dump(results, f)
Path(TMP_RESULTS_PATH).write_text(json.dumps(results))
packages/fileimport-service/requirements.txt
+1 -1
View File
@@ -1,2 +1,2 @@
numpy-stl==2.17.1
specklepy==2.9.0
specklepy==2.9.1
packages/fileimport-service/stl/import_file.py
+29 -23
View File
@@ -1,46 +1,50 @@
import json
import stl
from specklepy.objects.geometry import Mesh, Point
from specklepy.objects.geometry import Mesh
from specklepy.transports.server import ServerTransport
from specklepy.api.client import SpeckleClient
from specklepy.api.credentials import get_default_account
from specklepy.api import operations
import sys, os
TMP_RESULTS_PATH = '/tmp/import_result.json'
DEFAULT_BRANCH = 'uploads'
TMP_RESULTS_PATH = "/tmp/import_result.json"
DEFAULT_BRANCH = "uploads"
def import_stl():
file_path, user_id, stream_id, branch_name, commit_message = sys.argv[1:]
print(f'ImportSTL argv[1:]: {sys.argv[1:]}')
file_path, _, stream_id, branch_name, commit_message = sys.argv[1:]
print(f"ImportSTL argv[1:]: {sys.argv[1:]}")
# Parse input
stl_mesh = stl.mesh.Mesh.from_file(file_path)
print(f'Parsed mesh with {stl_mesh.points.shape[0]} faces ({stl_mesh.points.shape[0] * 3} vertices)')
print(
f"Parsed mesh with {stl_mesh.points.shape[0]} faces ({stl_mesh.points.shape[0] * 3} vertices)"
)
# Construct speckle obj
vertices = stl_mesh.points.flatten().tolist()
faces = []
for i in range(stl_mesh.points.shape[0]):
faces.extend([0, 3*i, 3*i+1, 3*i+2])
faces.extend([0, 3 * i, 3 * i + 1, 3 * i + 2])
speckle_mesh = Mesh(
vertices=vertices,
faces=faces,
colors=[],
textureCoordinates=[]
vertices=vertices, faces=faces, colors=[], textureCoordinates=[]
)
print('Constructed Speckle Mesh object')
print("Constructed Speckle Mesh object")
# Commit
client = SpeckleClient(host=os.getenv('SPECKLE_SERVER_URL', 'localhost:3000'), use_ssl=False)
client.authenticate(os.environ['USER_TOKEN'])
client = SpeckleClient(
host=os.getenv("SPECKLE_SERVER_URL", "localhost:3000"), use_ssl=False
)
client.authenticate_with_token(os.environ["USER_TOKEN"])
if not client.branch.get(stream_id, branch_name):
client.branch.create(stream_id, branch_name, 'File upload branch' if branch_name == 'uploads' else '')
client.branch.create(
stream_id,
branch_name,
"File upload branch" if branch_name == "uploads" else "",
)
transport = ServerTransport(client=client, stream_id=stream_id)
id = operations.send(
@@ -53,20 +57,22 @@ def import_stl():
stream_id=stream_id,
object_id=id,
branch_name=(branch_name or DEFAULT_BRANCH),
message=(commit_message or 'STL file upload'),
source_application='STL'
message=(commit_message or "STL file upload"),
source_application="STL",
)
return commit_id
if __name__ == '__main__':
if __name__ == "__main__":
from pathlib import Path
try:
commit_id = import_stl()
results = {'success': True, 'commitId': commit_id}
results = {"success": True, "commitId": commit_id}
except Exception as ex:
results = {'success': False, 'error': str(ex)}
print(ex)
with open(TMP_RESULTS_PATH, 'w') as f:
json.dump(results, f)
print(results)
Path(TMP_RESULTS_PATH).write_text(json.dumps(results))
packages/preview-service/bg_service/prometheusMetrics.js
+1 -1
View File
@@ -100,7 +100,7 @@ module.exports = {
metricDuration: new prometheusClient.Histogram({
name: 'speckle_server_operation_duration',
help: 'Summary of the operation durations in seconds',
buckets: [0.5, 1, 5, 10, 30, 60, 300, 600],
buckets: [0.5, 1, 5, 10, 30, 60, 300, 600, 1200, 1800],
labelNames: ['op']
}),
packages/preview-service/routes/preview.js
+6
View File
@@ -126,6 +126,12 @@ async function getScreenshot(objectUrl) {
}
router.get('/:streamId/:objectId', async function (req, res) {
const safeParamRgx = /^[\w]+$/i
const { streamId, objectId } = req.params || {}
if (!safeParamRgx.test(streamId) || !safeParamRgx.test(objectId)) {
return res.status(400).json({ error: 'Invalid streamId or objectId!' })
}
const objectUrl = `http://127.0.0.1:3001/streams/${req.params.streamId}/objects/${req.params.objectId}`
/*
let authToken = ''
packages/server/modules/activitystream/services/index.js
+1 -1
View File
@@ -40,7 +40,7 @@ module.exports = {
data: info
}
}
dispatchStreamEvent({
await dispatchStreamEvent({
streamId,
event: actionType,
eventPayload: webhooksPayload
packages/server/modules/auth/errors/index.ts
+6
View File
@@ -0,0 +1,6 @@
import { BaseError } from '@/modules/shared/errors'
export class InvalidAccessCodeRequestError extends BaseError {
static code = 'INVALID_ACCESS_CODE_REQUEST'
static defaultMessage = 'An issue occurred while generating an access code for an app'
}
packages/server/modules/auth/rest/index.js
+18 -3
View File
@@ -13,6 +13,8 @@ const {
const { validateToken, revokeTokenById } = require(`@/modules/core/services/tokens`)
const { revokeRefreshToken } = require(`@/modules/auth/services/apps`)
const { validateScopes } = require(`@/modules/shared`)
const { InvalidAccessCodeRequestError } = require('@/modules/auth/errors')
const { ForbiddenError } = require('apollo-server-errors')
// TODO: Secure these endpoints!
module.exports = (app) => {
@@ -24,14 +26,17 @@ module.exports = (app) => {
try {
const appId = req.query.appId
const app = await getApp({ id: appId })
if (!app) throw new Error('App does not exist.')
if (!app) throw new InvalidAccessCodeRequestError('App does not exist.')
const challenge = req.query.challenge
const userToken = req.query.token
if (!challenge) throw new InvalidAccessCodeRequestError('Missing challenge')
if (!userToken) throw new InvalidAccessCodeRequestError('Missing token')
// 1. Validate token
const { valid, scopes, userId } = await validateToken(userToken)
if (!valid) throw new Error('Invalid token')
if (!valid) throw new InvalidAccessCodeRequestError('Invalid token')
// 2. Validate token scopes
await validateScopes(scopes, 'tokens:write')
@@ -41,7 +46,17 @@ module.exports = (app) => {
} catch (err) {
sentry({ err })
debug('speckle:error')(err)
return res.status(400).send(err.message)
if (
err instanceof InvalidAccessCodeRequestError ||
err instanceof ForbiddenError
) {
return res.status(400).send(err.message)
} else {
return res
.status(500)
.send('Something went wrong while processing your request')
}
}
})
packages/server/modules/auth/strategies.js
+5 -1
View File
@@ -7,6 +7,7 @@ const passport = require('passport')
const sentry = require('@/logging/sentryHelper')
const { createAuthorizationCode } = require('./services/apps')
const { isSSLServer } = require('@/modules/shared/helpers/envHelper')
/**
* TODO: Get rid of session entirely, we don't use it for the app and it's not really necessary for the auth flow, so it only complicates things
@@ -24,7 +25,10 @@ module.exports = async (app) => {
secret: process.env.SESSION_SECRET,
saveUninitialized: false,
resave: false,
cookie: { maxAge: 1000 * 60 * 3 } // 3 minutes
cookie: {
maxAge: 1000 * 60 * 3, // 3 minutes
secure: isSSLServer()
}
})
/**
packages/server/modules/auth/strategies/local.js
+3 -15
View File
@@ -12,6 +12,7 @@ const {
finalizeInvitedServerRegistration,
resolveAuthRedirectPath
} = require('@/modules/serverinvites/services/inviteProcessingService')
const { getIpFromRequest } = require('@/modules/shared/utils/ip')
module.exports = async (app, session, sessionAppId, finalizeAuth) => {
const strategy = {
@@ -57,21 +58,8 @@ module.exports = async (app, session, sessionAppId, finalizeAuth) => {
if (!req.body.password) throw new Error('Password missing')
const user = req.body
user.ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress || ''
const ignorePrefixes = [
'192.168.',
'10.',
'127.',
'172.1',
'172.2',
'172.3',
'::'
]
for (const ipPrefix of ignorePrefixes)
if (user.ip.startsWith(ipPrefix)) {
delete user.ip
break
}
const ip = getIpFromRequest(req)
if (ip) user.ip = ip
if (
user.ip &&
!(await respectsLimits({ action: 'USER_CREATE', source: user.ip }))
packages/server/modules/blobstorage/index.js
+8
View File
@@ -31,6 +31,8 @@ const {
getFileSizeLimit
} = require('@/modules/blobstorage/services')
const { isArray } = require('lodash')
const {
NotFoundError,
ResourceMismatch,
@@ -176,6 +178,12 @@ exports.init = async (app) => {
allowAnonymousUsersOnPublicStreams
]),
async (req, res) => {
if (!isArray(req.body)) {
return res
.status(400)
.json({ error: 'An array of blob IDs expected in the body.' })
}
const bq = await getAllStreamBlobIds({ streamId: req.params.streamId })
const unknownBlobIds = req.body.filter(
(id) => bq.findIndex((bInfo) => bInfo.id === id) === -1
packages/server/modules/core/rest/diffDownload.js
+9
View File
@@ -7,6 +7,9 @@ const { contextMiddleware } = require('@/modules/shared')
const { validatePermissionsReadStream } = require('./authUtils')
const { SpeckleObjectsStream } = require('./speckleObjectsStream')
const { getObjectsStream } = require('../services/objects')
const {
rejectsRequestWithRatelimitStatusIfNeeded
} = require('@/modules/core/services/ratelimits')
const { pipeline, PassThrough } = require('stream')
@@ -14,6 +17,12 @@ module.exports = (app) => {
app.options('/api/getobjects/:streamId', cors())
app.post('/api/getobjects/:streamId', cors(), contextMiddleware, async (req, res) => {
const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({
action: 'POST /api/getobjects/:streamId',
req,
res
})
if (rejected) return rejected
const hasStreamAccess = await validatePermissionsReadStream(
req.params.streamId,
req
packages/server/modules/core/rest/diffUpload.js
+9
View File
@@ -5,6 +5,9 @@ const debug = require('debug')
const { contextMiddleware } = require('@/modules/shared')
const { validatePermissionsWriteStream } = require('./authUtils')
const {
rejectsRequestWithRatelimitStatusIfNeeded
} = require('@/modules/core/services/ratelimits')
const { hasObjects } = require('../services/objects')
@@ -12,6 +15,12 @@ module.exports = (app) => {
app.options('/api/diff/:streamId', cors())
app.post('/api/diff/:streamId', cors(), contextMiddleware, async (req, res) => {
const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({
action: 'POST /api/diff/:streamId',
req,
res
})
if (rejected) return rejected
const hasStreamAccess = await validatePermissionsWriteStream(
req.params.streamId,
req
packages/server/modules/core/rest/download.js
+17
View File
@@ -9,6 +9,9 @@ const { validatePermissionsReadStream } = require('./authUtils')
const { getObject, getObjectChildrenStream } = require('../services/objects')
const { SpeckleObjectsStream } = require('./speckleObjectsStream')
const { pipeline, PassThrough } = require('stream')
const {
rejectsRequestWithRatelimitStatusIfNeeded
} = require('@/modules/core/services/ratelimits')
module.exports = (app) => {
app.options('/objects/:streamId/:objectId', cors())
@@ -18,6 +21,13 @@ module.exports = (app) => {
cors(),
contextMiddleware,
async (req, res) => {
const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({
action: 'GET /objects/:streamId/:objectId',
req,
res
})
if (rejected) return rejected
const hasStreamAccess = await validatePermissionsReadStream(
req.params.streamId,
req
@@ -85,6 +95,13 @@ module.exports = (app) => {
cors(),
contextMiddleware,
async (req, res) => {
const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({
action: 'GET /objects/:streamId/:objectId/single',
req,
res
})
if (rejected) return rejected
const hasStreamAccess = await validatePermissionsReadStream(
req.params.streamId,
req
packages/server/modules/core/rest/upload.js
+10
View File
@@ -8,6 +8,9 @@ const { contextMiddleware } = require('@/modules/shared')
const { validatePermissionsWriteStream } = require('./authUtils')
const { createObjectsBatched } = require('../services/objects')
const {
rejectsRequestWithRatelimitStatusIfNeeded
} = require('@/modules/core/services/ratelimits')
const MAX_FILE_SIZE = 50 * 1024 * 1024
@@ -15,6 +18,13 @@ module.exports = (app) => {
app.options('/objects/:streamId', cors())
app.post('/objects/:streamId', cors(), contextMiddleware, async (req, res) => {
const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({
action: 'POST /objects/:streamId',
req,
res
})
if (rejected) return rejected
const hasStreamAccess = await validatePermissionsWriteStream(
req.params.streamId,
req
packages/server/modules/core/services/ratelimits.js
+38 -17
View File
@@ -25,7 +25,13 @@ const LIMITS = {
BRANCHES: parseInt(process.env.LIMIT_BRANCHES) || 1000, // per stream
TOKENS: parseInt(process.env.LIMIT_TOKENS) || 1000, // per user
ACTIVE_SUBSCRIPTIONS: parseInt(process.env.LIMIT_ACTIVE_SUBSCRIPTIONS) || 100, // per user
ACTIVE_CONNECTIONS: parseInt(process.env.LIMIT_ACTIVE_CONNECTIONS) || 100 // per source ip
ACTIVE_CONNECTIONS: parseInt(process.env.LIMIT_ACTIVE_CONNECTIONS) || 100, // per source ip
'POST /api/getobjects/:streamId': 200, // for 1 minute
'POST /api/diff/:streamId': 200, // for 1 minute
'POST /objects/:streamId': 200, // for 1 minute
'GET /objects/:streamId/:objectId': 200, // for 1 minute
'GET /objects/:streamId/:objectId/single': 200 // for 1 minute
}
const LIMIT_INTERVAL = {
@@ -42,7 +48,13 @@ const LIMIT_INTERVAL = {
BRANCHES: 0,
TOKENS: 0,
ACTIVE_SUBSCRIPTIONS: 0,
ACTIVE_CONNECTIONS: 0
ACTIVE_CONNECTIONS: 0,
'POST /api/getobjects/:streamId': 60,
'POST /api/diff/:streamId': 60,
'POST /objects/:streamId': 60,
'GET /objects/:streamId/:objectId': 60,
'GET /objects/:streamId/:objectId/single': 60
}
const rateLimitedCache = {}
@@ -74,22 +86,31 @@ async function shouldRateLimitNext({ action, source }) {
return shouldRateLimit
}
// returns true if the action is fine, false if it should be blocked because of exceeding limit
async function respectsLimits({ action, source }) {
const rateLimitKey = `${action} ${source}`
const promise = shouldRateLimitNext({ action, source }).then((shouldRateLimit) => {
if (shouldRateLimit) rateLimitedCache[rateLimitKey] = true
else delete rateLimitedCache[rateLimitKey]
})
if (rateLimitedCache[rateLimitKey]) {
await promise
}
if (rateLimitedCache[rateLimitKey]) limitsReached.labels(action).inc()
return !rateLimitedCache[rateLimitKey]
}
async function rejectsRequestWithRatelimitStatusIfNeeded({ action, req, res }) {
const source = req.context.userId || req.context.ip
if (!(await respectsLimits({ action, source })))
return res.status(429).set('X-Speckle-Meditation', 'https://http.cat/429').send({
err: 'You are sending too many requests. You have been rate limited. Please try again later.'
})
}
module.exports = {
LIMITS,
LIMIT_INTERVAL,
// returns true if the action is fine, false if it should be blocked because of exceeding limit
async respectsLimits({ action, source }) {
const rateLimitKey = `${action} ${source}`
const promise = shouldRateLimitNext({ action, source }).then((shouldRateLimit) => {
if (shouldRateLimit) rateLimitedCache[rateLimitKey] = true
else delete rateLimitedCache[rateLimitKey]
})
if (rateLimitedCache[rateLimitKey]) {
await promise
}
if (rateLimitedCache[rateLimitKey]) limitsReached.labels(action).inc()
return !rateLimitedCache[rateLimitKey]
}
respectsLimits,
rejectsRequestWithRatelimitStatusIfNeeded
}
packages/server/modules/index.js
+2 -1
View File
@@ -55,7 +55,8 @@ async function getSpeckleModules() {
'./blobstorage',
'./notifications',
'./activitystream',
'./accessrequests'
'./accessrequests',
'./webhooks'
]
for (const dir of moduleDirs) {
packages/server/modules/shared/helpers/envHelper.ts
+7
View File
@@ -56,3 +56,10 @@ export function shouldDisableNotificationsConsumption() {
process.env.DISABLE_NOTIFICATIONS_CONSUMPTION || 'false'
)
}
/**
* Check whether we're running an SSL server
*/
export function isSSLServer() {
return /^https:\/\//.test(getBaseUrl())
}
packages/server/modules/shared/index.js
+2 -1
View File
@@ -5,6 +5,7 @@ const { ForbiddenError, ApolloError } = require('apollo-server-express')
const { RedisPubSub } = require('graphql-redis-subscriptions')
const { buildRequestLoaders } = require('@/modules/core/loaders')
const { validateToken } = require(`@/modules/core/services/tokens`)
const { getIpFromRequest } = require('@/modules/shared/utils/ip')
const StreamPubsubEvents = Object.freeze({
UserStreamAdded: 'USER_STREAM_ADDED',
@@ -49,7 +50,7 @@ function addLoadersToCtx(ctx) {
async function buildContext({ req, connection }) {
// Parsing auth info
const ctx = await contextApiTokenHelper({ req, connection })
ctx.ip = getIpFromRequest(req)
// Adding request data loaders
return addLoadersToCtx(ctx)
}
packages/server/modules/shared/utils/ip.js
+15
View File
@@ -0,0 +1,15 @@
const getIpFromRequest = (req) => {
let ip
try {
ip = req.headers['cf-connecting-ip'] || req.ip || req.connection.remoteAddress || ''
} catch {
ip = ''
}
const ignorePrefixes = ['192.168.', '10.', '127.', '172.1', '172.2', '172.3', '::']
for (const ipPrefix of ignorePrefixes)
if (ip.startsWith(ipPrefix) || ip === '') return null
return ip
}
module.exports = { getIpFromRequest }
packages/server/modules/webhooks/index.js
View File
packages/server/modules/webhooks/index.ts
+27
View File
@@ -0,0 +1,27 @@
import cron from 'node-cron'
import { SpeckleModule } from '@/modules/shared/helpers/typeHelper'
import { modulesDebug } from '@/modules/shared/utils/logger'
import { scheduleExecution } from '@/modules/core/services/taskScheduler'
import { cleanOrphanedWebhookConfigs } from '@/modules/webhooks/services/cleanup'
const webhooksDebug = modulesDebug.extend('activities')
const scheduleWebhookCleanup = () => {
const cronExpression = '0 4 * * 1'
return scheduleExecution(cronExpression, 'weeklyWebhookCleanup', async () => {
webhooksDebug('Starting weekly webhooks cleanup')
await cleanOrphanedWebhookConfigs()
webhooksDebug('Finished cleanup')
})
}
let scheduledTask: cron.ScheduledTask | null = null
export const init: SpeckleModule['init'] = () => {
modulesDebug('🎣 Init webhooks module')
scheduledTask = scheduleWebhookCleanup()
}
export const shutdown: SpeckleModule['shutdown'] = () => {
if (scheduledTask) scheduledTask.stop()
}
packages/server/modules/webhooks/migrations/20221104104921_webhooks_drop_stream_fk.ts
+16
View File
@@ -0,0 +1,16 @@
import { Knex } from 'knex'
const TABLE_NAME = 'webhooks_config'
export async function up(knex: Knex): Promise<void> {
await knex.schema.alterTable(TABLE_NAME, (table) => {
// dropping the foreign key and not adding a new one
// if we still had the constraint, the late triggered event ie stream_delete hooks
// would not find the webhook configs
table.dropForeign('streamId')
})
}
export async function down() {
return
}
packages/server/modules/webhooks/services/cleanup.ts
+15
View File
@@ -0,0 +1,15 @@
import knex from '@/db/knex'
export async function cleanOrphanedWebhookConfigs() {
await knex.raw(
// i know im using a where in here, but this is used as background operation
`
delete from webhooks_config
where id in (
select wh.id from webhooks_config wh
left join streams st on wh."streamId" = st.id
where st.id is null
)
`
)
}
packages/server/modules/webhooks/services/webhooks.js
+2
View File
@@ -107,6 +107,8 @@ module.exports = {
}
}
// with this select, we must have the streamid available on the webhook config,
// even when the stream is deleted, to dispatch the stream deleted webhook events
const { rows } = await knex.raw(
`
SELECT * FROM webhooks_config WHERE "streamId" = ?
packages/server/modules/webhooks/tests/cleanup.spec.ts
+81
View File
@@ -0,0 +1,81 @@
import knex from '@/db/knex'
import { createStream } from '@/modules/core/services/streams'
import { createUser } from '@/modules/core/services/users'
import { cleanOrphanedWebhookConfigs } from '@/modules/webhooks/services/cleanup'
import { truncateTables } from '@/test/hooks'
import { expect } from 'chai'
import crs from 'crypto-random-string'
const WEBHOOKS_CONFIG_TABLE = 'webhooks_config'
const WEBHOOKS_EVENTS_TABLE = 'webhooks_events'
const WebhooksConfig = () => knex(WEBHOOKS_CONFIG_TABLE)
const randomId = () => crs({ length: 10 })
const countWebhooks = async () => {
const [{ count }] = await WebhooksConfig().count()
return parseInt(count as string)
}
describe('Webhooks cleanup @webhooks', () => {
before(async () => {
await truncateTables([WEBHOOKS_CONFIG_TABLE, WEBHOOKS_EVENTS_TABLE])
})
it('Cleans orphaned webhook configs', async () => {
const webhookConfig = {
id: randomId(),
streamId: randomId(),
url: 'foobar',
description: 'test_hook',
triggers: {
// eslint-disable-next-line camelcase
stream_update: true
}
}
await WebhooksConfig().insert(webhookConfig)
expect(await countWebhooks()).to.equal(1)
await cleanOrphanedWebhookConfigs()
expect(await countWebhooks()).to.equal(0)
})
it('Cleans orphans, leaves live ones intact', async () => {
const ownerId = await createUser({
name: 'User',
email: 'user@gmail.com',
password: 'jdsadjsadasfdsa'
})
const streamId = await createStream({
name: 'foo',
description: 'bar',
ownerId
})
const webhookConfigs = [
{
id: randomId(),
streamId: randomId(),
url: 'foobar',
description: 'test_hook',
triggers: {
// eslint-disable-next-line camelcase
stream_update: true
}
},
{
id: randomId(),
streamId,
url: 'foobar',
description: 'test_hook',
triggers: {
// eslint-disable-next-line camelcase
stream_update: true
}
}
]
await Promise.all(webhookConfigs.map((c) => WebhooksConfig().insert(c)))
expect(await countWebhooks()).to.equal(2)
await cleanOrphanedWebhookConfigs()
expect(await countWebhooks()).to.equal(1)
})
})
packages/viewer-sandbox/src/Sandbox.ts
+20 -20
View File
@@ -503,27 +503,27 @@ export default class Sandbox {
this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams
this.viewer.requestRender()
})
staticAoFolder
.addInput(Sandbox.pipelineParams.staticAoParams, 'minDistance', {
min: 0,
max: 100,
step: 0.000001
})
.on('change', () => {
this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams
this.viewer.requestRender()
})
// staticAoFolder
// .addInput(Sandbox.pipelineParams.staticAoParams, 'minDistance', {
// min: 0,
// max: 100,
// step: 0.000001
// })
// .on('change', () => {
// this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams
// this.viewer.requestRender()
// })
staticAoFolder
.addInput(Sandbox.pipelineParams.staticAoParams, 'maxDistance', {
min: 0,
max: 100,
step: 0.000001
})
.on('change', () => {
this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams
this.viewer.requestRender()
})
// staticAoFolder
// .addInput(Sandbox.pipelineParams.staticAoParams, 'maxDistance', {
// min: 0,
// max: 100,
// step: 0.000001
// })
// .on('change', () => {
// this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams
// this.viewer.requestRender()
// })
staticAoFolder
.addInput(Sandbox.pipelineParams.staticAoParams, 'kernelRadius', {
min: 0,
packages/viewer-sandbox/src/main.ts
+6
View File
@@ -129,4 +129,10 @@ await sandbox.loadUrl(
// 'https://speckle.xyz/streams/b85d53c3b4/commits/b47f21b707'
// Crankshaft
// 'https://speckle.xyz/streams/c239718aff/commits/b3a8cfb97d'
// Building AO params
// 'https://latest.speckle.dev/streams/0dd74866d0/commits/317e210afa'
// Murder Cube
// 'https://latest.speckle.dev/streams/c1faab5c62/commits/7f0c4d2fc1/'
// Classroom
// 'https://speckle.xyz/streams/0208ffb67b/commits/a980292728'
)
packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts
+4 -2
View File
@@ -264,6 +264,8 @@ export const speckleStaticAoGenerateFrag = /* glsl */ `
mat3 kernelMatrix = mat3( tangent, bitangent, viewNormal );
float occlusion = 0.0;
float kernelSize_ws = computeKernelSize(-viewPosition.z, kernelRadius);
float div = float( KERNEL_SIZE);
float maxDist = kernelSize_ws / (cameraFar - cameraNear);
for ( int i = 0; i < KERNEL_SIZE; i ++ ) {
vec3 sampleVector = kernelMatrix * kernel[ i ]; // reorient sample vector in view space
vec3 samplePoint = viewPosition + ( sampleVector * kernelSize_ws ); // calculate sample point
@@ -273,11 +275,11 @@ export const speckleStaticAoGenerateFrag = /* glsl */ `
float realDepth = getLinearDepth( samplePointUv ); // get linear depth from depth texture
float sampleDepth = viewZToOrthographicDepth( samplePoint.z + bias, cameraNear, cameraFar ); // compute linear depth of the sample view Z value
float delta = sampleDepth - realDepth;
if ( delta > minDistance && delta < maxDistance ) { // if fragment is before sample point, increase occlusion
if ( delta > 0. && delta < maxDist ) { // if fragment is before sample point, increase occlusion
occlusion += 1.0;
}
}
return clamp( occlusion * intensity / float( KERNEL_SIZE ), 0.0, 1.0 );
return clamp( occlusion * intensity / div, 0.0, 1.0 );
#endif
}
void main() {
utils/helm/speckle-server/templates/server/deployment.yml
+2 -2
View File
@@ -70,7 +70,7 @@ spec:
command:
- node
- -e
- require('request')('http://localhost:3000/graphql?query={serverInfo{version}}', (e,r,b) => process.exit(b.toLowerCase().includes('error')))
- "require('request')({headers: {'Content-Type': 'application/json'}, uri: 'http://localhost:3000/graphql?query={serverInfo{version}}', method: 'GET' }, (e,r,b) => process.exit(r.statusCode != 200 || b.toLowerCase().includes('error')))"
readinessProbe:
initialDelaySeconds: 5
@@ -80,7 +80,7 @@ spec:
command:
- node
- -e
- require('request')('http://localhost:3000/graphql?query={serverInfo{version}}', (e,r,b) => process.exit(b.toLowerCase().includes('error')))
- "require('request')({headers: {'Content-Type': 'application/json'}, uri: 'http://localhost:3000/graphql?query={serverInfo{version}}', method: 'GET' }, (e,r,b) => process.exit(r.statusCode != 200 || b.toLowerCase().includes('error')))"
env:
- name: CANONICAL_URL
utils/helm/speckle-server/templates/tests/deployment.yml
+41 -37
View File
@@ -1,49 +1,53 @@
{{- if .Values.helm_test_enabled }}
apiVersion: v1
kind: Pod
apiVersion: batch/v1
kind: Job
metadata:
name: "speckle-test-deployment"
name: "speckle-test"
namespace: {{ .Values.namespace }}
annotations:
"helm.sh/hook": test
helm.sh/hook: test
labels:
{{ include "test.labels" . | indent 4 }}
spec:
containers:
- name: test-deployment
image: speckle/speckle-test-deployment:{{ .Values.docker_image_tag }}
env:
- name: SPECKLE_SERVER
value: https://{{ .Values.domain }}
- name: SERVER_VERSION
value: {{ .Values.docker_image_tag }}
resources:
requests:
cpu: {{ .Values.test.requests.cpu }}
memory: {{ .Values.test.requests.memory }}
limits:
cpu: {{ .Values.test.limits.cpu }}
memory: {{ .Values.test.limits.memory }}
backoffLimit: 1
parallelism: 1
completions: 1
template:
spec:
containers:
- name: test-deployment
image: speckle/speckle-test-deployment:{{ .Values.docker_image_tag }}
env:
- name: SPECKLE_SERVER
value: https://{{ .Values.domain }}
- name: SERVER_VERSION
value: {{ .Values.docker_image_tag }}
resources:
requests:
cpu: {{ .Values.test.requests.cpu }}
memory: {{ .Values.test.requests.memory }}
limits:
cpu: {{ .Values.test.limits.cpu }}
memory: {{ .Values.test.limits.memory }}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 20000
restartPolicy: Never
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 20000
runAsGroup: 30000
seccompProfile:
type: RuntimeDefault
restartPolicy: Never
securityContext:
runAsNonRoot: true
runAsUser: 20000
runAsGroup: 30000
seccompProfile:
type: RuntimeDefault
{{- if .Values.test.serviceAccount.create }}
serviceAccountName: {{ include "test.name" $ }}
{{- end }}
{{- if .Values.test.serviceAccount.create }}
serviceAccountName: {{ include "test.name" $ }}
{{- end }}
{{- end }}