huanld/speckle-server
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'main' into 773-optimized-embed-endpoint

Browse Source
This commit is contained in:
Dimitrie Stefanescu
2022-06-19 20:00:29 +01:00
parent 5e18f5610f ed98c5da14
commit df29ab89bf
47 changed files with 2416 additions and 11442 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+11 -2
View File
@@ -115,11 +115,15 @@ jobs:
docker:
- image: cimg/node:16.15
- image: cimg/redis:6.2.6
- image: 'cimg/postgres:12.8'
- image: 'cimg/postgres:14.2'
environment:
POSTGRES_DB: speckle2_test
POSTGRES_PASSWORD: speckle
POSTGRES_USER: speckle
- image: 'minio/minio'
command: server /data --console-address ":9001"
# environment:
environment:
NODE_ENV: test
DATABASE_URL: 'postgres://speckle:speckle@localhost:5432/speckle2_test'
@@ -128,6 +132,11 @@ jobs:
SESSION_SECRET: 'keyboard cat'
STRATEGY_LOCAL: 'true'
CANONICAL_URL: 'http://localhost:3000'
S3_ENDPOINT: 'http://localhost:9000'
S3_ACCESS_KEY: 'minioadmin'
S3_SECRET_KEY: 'minioadmin'
S3_BUCKET: 'speckle-server'
S3_CREATE_BUCKET: 'true'
steps:
- checkout
- restore_cache:
@@ -146,7 +155,7 @@ jobs:
- .yarn/cache
- .yarn/unplugged
- run: 'dockerize -wait tcp://localhost:5432 -timeout 1m'
- run: 'dockerize -wait tcp://localhost:5432 -wait tcp://localhost:6379 -timeout 1m'
- run:
command: touch .env.test
.dockerignore
+1
View File
@@ -12,6 +12,7 @@ ISSUE_TEMPLATE.md
.mocharc.js
readme.md
**/Dockerfile
**/.venv
.pnp.*
.yarn/*
.gitignore
+2
View File
@@ -23,6 +23,8 @@ packages/frontend/schema.graphql
packages/server/reports*
**/venv/**
**/.venv/**
*.pyc
**/start/
# Profiler output
.pre-commit-config.yaml
+2
View File
@@ -12,6 +12,8 @@ repos:
- eslint@8.11.0
- eslint-config-prettier@8.5.0
- eslint-plugin-vue@8.5.0
- '@babel/eslint-parser@7.18.2'
- '@babel/preset-env@t 7.16.11'
- '@typescript-eslint/eslint-plugin@5.21.0'
- '@typescript-eslint/parser@5.21.0'
- typescript@4.5.4
.prettierignore
+4 -1
View File
@@ -10,6 +10,7 @@ packages/viewer/example/speckleviewer.web.js
packages/frontend/**/generated
package-lock.json
yarn.lock
.yarn
# Profiler output
events.json
@@ -18,4 +19,6 @@ events.json
utils/helm/speckle-server/templates
# Optional eslint cache
.eslintcache
.eslintcache
.venv
venv
packages/fileimport-service/ifc/index.js
+1 -1
View File
@@ -1,4 +1,4 @@
const fetch = require('node-fetch')
const { fetch } = require('undici')
const Parser = require('./parser')
const ServerAPI = require('./api.js')
packages/fileimport-service/knex.js
+9 -3
View File
@@ -1,9 +1,15 @@
/* eslint-disable camelcase */
'use strict'
module.exports = require('knex')({
client: 'pg',
connection:
process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@localhost/speckle',
pool: { min: 1, max: 1 }
connection: {
application_name: 'speckle_fileimport_service',
connectionString:
process.env.PG_CONNECTION_STRING ||
'postgres://speckle:speckle@localhost/speckle',
query_timeout: 4.32e7
},
pool: { min: 0, max: 1 }
// migrations are in managed in the server package
})
packages/fileimport-service/package.json
+4 -3
View File
@@ -15,7 +15,8 @@
"node": "^16.0.0"
},
"scripts": {
"dev": "cross-env S3_BUCKET=speckle-server POSTGRES_URL=postgres://speckle:speckle@localhost/speckle NODE_ENV=development SPECKLE_SERVER_URL=localhost:3000 nodemon ./src/daemon.js",
"dev": "cross-env S3_BUCKET=speckle-server POSTGRES_URL=postgres://speckle:speckle@localhost/speckle NODE_ENV=development SPECKLE_SERVER_URL=http://localhost:3000 nodemon ./src/daemon.js",
"parse:ifc": "node ./ifc/import_file.js /tmp/file_to_import/file 33763848d6 2e4bfb467a main File upload: steelplates.ifc",
"lint": "eslint . --ext .js,.ts"
},
"bugs": {
@@ -26,11 +27,11 @@
"bcrypt": "^5.0.1",
"crypto-random-string": "^3.3.1",
"knex": "^2.0.0",
"node-fetch": "^2.6.5",
"pg": "^8.7.3",
"prom-client": "^14.0.1",
"undici": "^5.4.0",
"valid-filename": "^3.1.0",
"web-ifc": "^0.0.33"
"web-ifc": "^0.0.35"
},
"devDependencies": {
"cross-env": "^7.0.3",
packages/fileimport-service/src/daemon.js
+12 -20
View File
@@ -8,8 +8,9 @@ const {
metricOperationErrors
} = require('./prometheusMetrics')
const knex = require('../knex')
const FileUploads = () => knex('file_uploads')
const { getFileStream } = require('./filesApi')
const { downloadFile } = require('./filesApi')
const fs = require('fs')
const { spawn } = require('child_process')
@@ -51,17 +52,7 @@ async function doTask(task) {
const metricDurationEnd = metricDuration.startTimer()
try {
console.log('Doing task ', task)
const { rows } = await knex.raw(
`
SELECT
id as "fileId", "streamId", "branchName", "userId", "fileName", "fileType", "fileSize"
FROM file_uploads
WHERE id = ?
LIMIT 1
`,
[task.id]
)
const info = rows[0]
const info = await FileUploads().where({ id: task.id }).first()
if (!info) {
throw new Error('Internal error: DB inconsistent')
}
@@ -70,13 +61,6 @@ async function doTask(task) {
fs.mkdirSync(TMP_INPUT_DIR, { recursive: true })
const upstreamFileStream = await getFileStream({ fileId: info.fileId })
const diskFileStream = fs.createWriteStream(TMP_FILE_PATH)
upstreamFileStream.pipe(diskFileStream)
await new Promise((fulfill) => diskFileStream.on('finish', fulfill))
serverApi = new ServerAPI({ streamId: info.streamId })
const { token } = await serverApi.createToken({
userId: info.userId,
@@ -86,6 +70,13 @@ async function doTask(task) {
})
tempUserToken = token
await downloadFile({
fileId: info.id,
streamId: info.streamId,
token,
destination: TMP_FILE_PATH
})
if (info.fileType === 'ifc') {
await runProcessWithTimeout(
'node',
@@ -122,7 +113,8 @@ async function doTask(task) {
await objDependencies.downloadDependencies({
objFilePath: TMP_FILE_PATH,
streamId: info.streamId,
destinationDir: TMP_INPUT_DIR
destinationDir: TMP_INPUT_DIR,
token: tempUserToken
})
await runProcessWithTimeout(
packages/fileimport-service/src/filesApi.js
+25 -29
View File
@@ -1,36 +1,32 @@
/* istanbul ignore file */
'use strict'
const S3 = require('aws-sdk/clients/s3')
function getS3Config() {
// TODO: use ENV
return {
accessKeyId: process.env.S3_ACCESS_KEY || 'minioadmin',
secretAccessKey: process.env.S3_SECRET_KEY || 'minioadmin',
endpoint: process.env.S3_ENDPOINT || 'http://127.0.0.1:9000',
s3ForcePathStyle: true,
signatureVersion: 'v4'
}
}
const fs = require('fs')
const path = require('node:path')
const { stream, fetch } = require('undici')
module.exports = {
async getFileStream({ fileId }) {
const s3 = new S3(getS3Config())
const Bucket = process.env.S3_BUCKET
const Key = `files/${fileId}`
const fileStream = s3.getObject({ Key, Bucket }).createReadStream()
return fileStream
async downloadFile({ fileId, streamId, token, destination }) {
fs.mkdirSync(path.dirname(destination), { recursive: true })
await stream(
`${process.env.SPECKLE_SERVER_URL}/api/stream/${streamId}/blob/${fileId}`,
{
opaque: fs.createWriteStream(destination),
headers: {
Authorization: `Bearer ${token}`
}
},
({ opaque }) => opaque
)
},
async readFile({ fileId }) {
const s3 = new S3(getS3Config())
const Bucket = process.env.S3_BUCKET
const Key = `files/${fileId}`
const s3Data = await s3.getObject({ Key, Bucket }).promise()
return s3Data.Body
async getFileInfoByName({ fileName, streamId, token }) {
const response = await fetch(
`${process.env.SPECKLE_SERVER_URL}/api/stream/${streamId}/blobs?fileName=${fileName}`,
{
headers: {
Authorization: `Bearer ${token}`
}
}
)
return response.json()
}
}
packages/fileimport-service/src/filesMetadata.js
-18
View File
@@ -1,18 +0,0 @@
const knex = require('../knex')
module.exports = {
async getFileInfoByName({ streamId, fileName }) {
const { rows } = await knex.raw(
`
SELECT
id as "fileId", "streamId", "branchName", "userId", "fileName", "fileType"
FROM file_uploads
WHERE "streamId" = ? AND "fileName" = ?
ORDER BY "uploadDate" DESC
LIMIT 1
`,
[streamId, fileName]
)
return rows[0]
}
}
packages/fileimport-service/src/objDependencies.js
+40 -46
View File
@@ -4,61 +4,55 @@ const fs = require('fs')
const readline = require('readline')
const path = require('path')
const { getFileInfoByName } = require('./filesMetadata')
const { getFileStream } = require('./filesApi')
const { downloadFile, getFileInfoByName } = require('./filesApi')
const isValidFilename = require('valid-filename')
async function tryDownloadFile({ fileName, streamId, destinationDir }) {
if (!isValidFilename(fileName)) {
console.log(`Invalid filename reference in OBJ dependencies: ${fileName}`)
return false
const getReferencedMtlFiles = async ({ objFilePath }) => {
const mtlFiles = []
try {
const rl = readline.createInterface({
input: fs.createReadStream(objFilePath),
crlfDelay: Infinity
})
rl.on('line', (line) => {
if (line.startsWith('mtllib ')) {
const mtlFile = line.slice('mtllib '.length).trim()
mtlFiles.push(mtlFile)
}
})
await events.once(rl, 'close')
} catch (err) {
console.error(`Error getting dependencies for file ${objFilePath}: ${err}`)
}
const fileInfo = await getFileInfoByName({ streamId, fileName })
if (!fileInfo) {
console.log(`OBJ dependency file not found in stream: ${fileName}`)
return false
}
const filePath = path.join(destinationDir, fileName)
const upstreamFileStream = await getFileStream({ fileId: fileInfo.fileId })
const diskFileStream = fs.createWriteStream(filePath)
upstreamFileStream.pipe(diskFileStream)
await new Promise((fulfill) => diskFileStream.on('finish', fulfill))
return true
return mtlFiles
}
module.exports = {
async getReferencedMtlFiles({ objFilePath }) {
const mtlFiles = []
async downloadDependencies({ objFilePath, streamId, destinationDir, token }) {
const dependencies = await getReferencedMtlFiles({ objFilePath })
try {
const rl = readline.createInterface({
input: fs.createReadStream(objFilePath),
crlfDelay: Infinity
})
rl.on('line', (line) => {
if (line.startsWith('mtllib ')) {
const mtlFile = line.slice('mtllib '.length).trim()
mtlFiles.push(mtlFile)
}
})
await events.once(rl, 'close')
} catch (err) {
console.error(`Error getting dependencies for file ${objFilePath}: ${err}`)
}
return mtlFiles
},
async downloadDependencies({ objFilePath, streamId, destinationDir }) {
const dependencies = await this.getReferencedMtlFiles({ objFilePath })
console.log(`Obj file depends on ${dependencies}`)
for (const mtlFile of dependencies) {
await tryDownloadFile({ fileName: mtlFile, streamId, destinationDir })
// there might be multiple files named with the same name, take the first...
const [file] = (await getFileInfoByName({ fileName: mtlFile, streamId, token }))
.blobs
if (!file) {
console.log(`OBJ dependency file not found in stream: ${mtlFile}`)
continue
}
if (!isValidFilename(mtlFile)) {
console.log(`Invalid filename reference in OBJ dependencies: ${mtlFile}`)
continue
}
await downloadFile({
fileId: file.id,
streamId,
token,
destination: path.join(destinationDir, mtlFile)
})
}
}
}
packages/frontend/src/embed/EmbedViewer.vue
+3
View File
@@ -259,9 +259,12 @@ body::-webkit-scrollbar {
.spinning-icon {
animation: spinner-spin 0.5s linear infinite;
}
<<<<<<< HEAD
}
.no-mouse {
pointer-events: none;
=======
>>>>>>> main
}
.mouse {
pointer-events: auto;
packages/frontend/src/embed/EmbedViewerCore.vue
+4
View File
@@ -218,7 +218,11 @@ export default Vue.extend({
top: 0;
left: 0;
<<<<<<< HEAD
z-index: 1;
=======
z-index: 10;
>>>>>>> main
}
.no-scrollbar {
packages/frontend/src/main/components/stream/uploads/FileProcessingItem.vue
+7 -4
View File
@@ -119,11 +119,14 @@ export default {
},
methods: {
async downloadOriginalFile() {
const res = await fetch(`/api/file/${this.fileId}`, {
headers: {
Authorization: localStorage.getItem('AuthToken')
const res = await fetch(
`/api/stream/${this.$route.params.streamId}/blob/${this.fileId}`,
{
headers: {
Authorization: localStorage.getItem('AuthToken')
}
}
})
)
const blob = await res.blob()
const file = window.URL.createObjectURL(blob)
packages/frontend/src/main/components/stream/uploads/FileUploadItem.vue
+2 -5
View File
@@ -78,9 +78,6 @@ export default {
'progress',
function (e) {
this.percentCompleted = (e.loaded / e.total) * 100
if (this.percentCompleted >= 100) {
this.$emit('done', this.file.name)
}
}.bind(this)
)
@@ -88,7 +85,7 @@ export default {
request.addEventListener(
'load',
function () {
if (request.status !== 200) {
if (request.status !== 201) {
this.error = request.response
}
@@ -99,7 +96,7 @@ export default {
request.addEventListener(
'error',
function () {
if (request.status !== 200) {
if (request.status !== 201) {
this.error = request.response
}
}.bind(this)
packages/preview-service/knex.js
+9 -3
View File
@@ -1,9 +1,15 @@
/* eslint-disable camelcase */
'use strict'
module.exports = require('knex')({
client: 'pg',
connection:
process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@localhost/speckle',
pool: { min: 1, max: 2 }
connection: {
application_name: 'speckle_preview_service',
connectionString:
process.env.PG_CONNECTION_STRING ||
'postgres://speckle:speckle@localhost/speckle',
query_timeout: 4.32e7
},
pool: { min: 0, max: 2 }
// migrations are in managed in the server package
})
packages/preview-service/package-lock.json
Generated
-10979
View File
File diff suppressed because it is too large Load Diff
packages/server/.vscode/launch.json
Vendored
+3 -3
View File
@@ -5,11 +5,11 @@
"version": "0.2.0",
"configurations": [
{
"name": "Launch via NPM",
"name": "Launch via YARN",
"request": "launch",
"console": "integratedTerminal",
"runtimeArgs": ["run-script", "dev"],
"runtimeExecutable": "npm",
"runtimeArgs": ["dev"],
"runtimeExecutable": "yarn",
"skipFiles": ["<node_internals>/**"],
"type": "pwa-node",
"envFile": "${workspaceFolder}/.env"
packages/server/app.js
+4 -5
View File
@@ -8,7 +8,6 @@ const express = require('express')
require('express-async-errors')
const compression = require('compression')
const logger = require('morgan-debug')
const bodyParser = require('body-parser')
const debug = require('debug')
const { createTerminus } = require('@godaddy/terminus')
@@ -106,15 +105,15 @@ exports.init = async () => {
app.use(compression())
}
app.use(bodyParser.json({ limit: '100mb' }))
app.use(bodyParser.urlencoded({ limit: '100mb', extended: false }))
app.use(express.json({ limit: '100mb' }))
app.use(express.urlencoded({ limit: '100mb', extended: false }))
const { init } = require('./modules')
// Initialise default modules, including rest api handlers
// Initialize default modules, including rest api handlers
await init(app)
// Initialise graphql server
// Initialize graphql server
graphqlServer = module.exports.buildApolloServer()
graphqlServer.applyMiddleware({ app })
packages/server/knexfile.js
+31 -16
View File
@@ -1,3 +1,4 @@
/* eslint-disable camelcase */
/* istanbul ignore file */
'use strict'
@@ -44,30 +45,44 @@ if (env.POSTGRES_USER && env.POSTGRES_PASSWORD) {
// types.setTypeParser(TIMESTAMPTZ_OID, (val) => val)
// types.setTypeParser(TIMESTAMP_OID, (val) => val)
// Another NOTE:
// this is why the new datetime columns are created like this
// table.specificType('createdAt', 'TIMESTAMPTZ(3)').defaultTo(knex.fn.now())
const postgresMaxConnections = parseInt(env.POSTGRES_MAX_CONNECTIONS_SERVER) || 4
const commonConfig = {
client: 'pg',
migrations: {
directory: migrationDirs
},
pool: { min: 0, max: postgresMaxConnections }
}
/** @type {Object<string, import('knex').Knex.Config>} */
const config = {
test: {
client: 'pg',
connection: connectionUri || 'postgres://localhost/speckle2_test',
migrations: {
directory: migrationDirs
...commonConfig,
connection: {
connectionString: connectionUri || 'postgres://localhost/speckle2_test',
application_name: 'speckle_server'
}
},
development: {
client: 'pg',
connection: connectionUri || 'postgres://localhost/speckle2_dev',
migrations: {
directory: migrationDirs
},
pool: { min: 2, max: 4 }
...commonConfig,
connection: {
connectionString: connectionUri || 'postgres://localhost/speckle2_dev',
application_name: 'speckle_server'
}
},
production: {
client: 'pg',
connection: connectionUri,
migrations: {
directory: migrationDirs
},
pool: { min: 2, max: 4 }
...commonConfig,
connection: {
connectionString: connectionUri,
application_name: 'speckle_server',
// global timeout of 12 hours, that kills stuck connections
query_timeout: 4.32e7
}
}
}
packages/server/modules/blobstorage/graph/resolvers/index.js
+42
View File
@@ -0,0 +1,42 @@
const {
getBlobMetadata,
getBlobMetadataCollection,
blobCollectionSummary
} = require('@/modules/blobstorage/services')
const { NotFoundError, ResourceMismatch } = require('@/modules/shared/errors')
const { UserInputError } = require('apollo-server-errors')
module.exports = {
Stream: {
async blobs(parent, args) {
const streamId = parent.id
const [summary, blobs] = await Promise.all([
blobCollectionSummary({
streamId,
query: args.query
}),
getBlobMetadataCollection({
streamId,
query: args.query,
limit: args.limit,
cursor: args.cursor
})
])
return {
totalCount: summary.totalCount,
totalSize: summary.totalSize,
cursor: blobs.cursor,
items: blobs.blobs
}
},
async blob(parent, args) {
try {
return await getBlobMetadata({ streamId: parent.id, blobId: args.id })
} catch (err) {
if (err instanceof NotFoundError) return null
if (err instanceof ResourceMismatch) throw new UserInputError(err.message)
throw err
}
}
}
}
packages/server/modules/blobstorage/graph/schemas/blobstorage.graphql
+27
View File
@@ -0,0 +1,27 @@
extend type Stream {
"""
Get the metadata collection of blobs stored for this stream.
"""
blobs(query: String, limit: Int = 25, cursor: String): BlobMetadataCollection
blob(id: String!): BlobMetadata
}
type BlobMetadataCollection {
totalCount: Int!
totalSize: Int!
cursor: String
items: [BlobMetadata!]
}
type BlobMetadata {
id: String!
streamId: String!
userId: String!
fileName: String!
fileType: String!
fileSize: Int
uploadStatus: Int!
uploadError: String
createdAt: DateTime!
}
packages/server/modules/blobstorage/index.js
+207
View File
@@ -0,0 +1,207 @@
const debug = require('debug')
const { contextMiddleware } = require('@/modules/shared')
const Busboy = require('busboy')
const {
authMiddlewareCreator,
streamReadPermissions,
streamWritePermissions,
allowForAllRegisteredUsersOnPublicStreamsWithPublicComments
} = require('@/modules/shared/authz')
const {
ensureStorageAccess,
storeFileStream,
getObjectStream,
deleteObject,
getObjectAttributes
} = require('@/modules/blobstorage/objectStorage')
const crs = require('crypto-random-string')
const {
uploadFileStream,
getFileStream,
markUploadError,
markUploadSuccess,
markUploadOverFileSizeLimit,
deleteBlob,
getBlobMetadata,
getBlobMetadataCollection
} = require('@/modules/blobstorage/services')
const { NotFoundError, ResourceMismatch } = require('@/modules/shared/errors')
const ensureConditions = async () => {
if (process.env.DISABLE_FILE_UPLOADS) {
debug('speckle:modules')('📦 Blob storage is DISABLED')
return
} else {
debug('speckle:modules')('📦 Init BlobStorage module')
await ensureStorageAccess()
}
if (!process.env.S3_BUCKET) {
debug('speckle:error')(
'S3_BUCKET env variable was not specified. 📦 BlobStorage will be DISABLED.'
)
return
}
}
const errorHandler = async (req, res, callback) => {
try {
await callback(req, res)
} catch (err) {
if (err instanceof NotFoundError) {
res.status(404).send({ error: err.message })
} else if (err instanceof ResourceMismatch) {
res.status(400).send({ error: err.message })
} else {
res.status(500).send({ error: err.message })
}
}
}
exports.init = async (app) => {
await ensureConditions()
// eslint-disable-next-line no-unused-vars
app.post(
'/api/stream/:streamId/blob',
contextMiddleware,
authMiddlewareCreator([
...streamWritePermissions,
// todo should we add public comments upload escape hatch?
allowForAllRegisteredUsersOnPublicStreamsWithPublicComments
]),
async (req, res) => {
// no checking of startup conditions, just dont init the endpoints if not configured right
//authorize request
const uploadOperations = {}
const finalizePromises = []
const busboy = Busboy({
headers: req.headers,
// this is 100 MB which matches the current frontend file size limit
limits: { fileSize: 104_857_600 }
})
const streamId = req.params.streamId
busboy.on('file', (name, file, info) => {
const { filename: fileName } = info
const fileType = fileName.split('.').pop().toLowerCase()
const blobId = crs({ length: 10 })
uploadOperations[blobId] = uploadFileStream(
storeFileStream,
{ streamId, userId: req.context.userId },
{ blobId, fileName, fileType, fileStream: file }
)
//this file level 'close' is fired when a single file upload finishes
//this way individual upload statuses can be updated, when done
file.on('close', async () => {
//this is handled by the file.on('limit', ...) event
if (file.truncated) return
await uploadOperations[blobId]
finalizePromises.push(
markUploadSuccess(getObjectAttributes, streamId, blobId)
)
})
file.on('limit', () => {
finalizePromises.push(
markUploadOverFileSizeLimit(deleteObject, streamId, blobId)
)
})
file.on('error', (err) => {
console.log(err)
finalizePromises.push(
markUploadError(deleteObject, blobId, 'i need some error info here')
)
})
})
busboy.on('finish', async () => {
// make sure all upload operations have been awaited,
// otherwise the finish even can fire before all async operations finish
//resulting in missing return values
await Promise.all(Object.values(uploadOperations))
// have to make sure all finalize promises have been awaited
const uploadResults = await Promise.all(finalizePromises)
res.status(201).send({ uploadResults })
})
busboy.on('error', (err) => {
debug('speckle:error')(`File upload error: ${err}`)
const status = 400
const response = 'Upload request error. The server logs have more details'
res.status(status).end(response)
})
req.pipe(busboy)
}
)
app.get(
'/api/stream/:streamId/blob/:blobId',
contextMiddleware,
authMiddlewareCreator([
...streamReadPermissions,
allowForAllRegisteredUsersOnPublicStreamsWithPublicComments
]),
async (req, res) => {
errorHandler(req, res, async (req, res) => {
const { fileName } = await getBlobMetadata({
streamId: req.params.streamId,
blobId: req.params.blobId
})
const fileStream = await getFileStream({
getObjectStream,
streamId: req.params.streamId,
blobId: req.params.blobId
})
res.writeHead(200, {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${fileName}"`
})
fileStream.pipe(res)
})
}
)
app.delete(
'/api/stream/:streamId/blob/:blobId',
contextMiddleware,
authMiddlewareCreator(streamWritePermissions),
async (req, res) => {
errorHandler(req, res, async (req, res) => {
await deleteBlob({
streamId: req.params.streamId,
blobId: req.params.blobId,
deleteObject
})
res.status(204).send()
})
}
)
app.get(
'/api/stream/:streamId/blobs',
contextMiddleware,
authMiddlewareCreator(streamWritePermissions),
async (req, res) => {
const fileName = req.query.fileName
errorHandler(req, res, async (req, res) => {
const blobMetadataCollection = await getBlobMetadataCollection({
streamId: req.params.streamId,
query: fileName
})
res.status(200).send(blobMetadataCollection)
})
}
)
app.delete(
'/api/stream/:streamId/blobs',
contextMiddleware,
authMiddlewareCreator(streamWritePermissions)
// async (req, res) => {}
)
}
exports.finalize = () => {}
packages/server/modules/blobstorage/migrations/202206030936_add_asset_storage.js
+31
View File
@@ -0,0 +1,31 @@
/* istanbul ignore file */
const TABLE_NAME = 'blob_storage'
/**
* @param { import("knex").Knex } knex
* @returns { Promise<void> }
*/
exports.up = async (knex) => {
await knex.schema.createTable(TABLE_NAME, (table) => {
table.string('id', 10)
// dont cascade on delete, cause it doesn't clean the object storage for the objs
// it needs to be exposed as a service, to be able to cleanup fully after a stream
table.string('streamId', 10)
table.string('userId', 10)
table.string('objectKey')
table.string('fileName').notNullable()
table.string('fileType').notNullable()
table.integer('fileSize')
// 0 = uploading, 1 = success, 2 = error
table.integer('uploadStatus').notNullable().defaultTo(0)
table.string('uploadError')
table.specificType('createdAt', 'TIMESTAMPTZ(3)').defaultTo(knex.fn.now())
table.primary(['id', 'streamId'])
})
}
exports.down = async (knex) => {
await knex.schema.dropTableIfExists(TABLE_NAME)
}
packages/server/modules/blobstorage/objectStorage.js
+117
View File
@@ -0,0 +1,117 @@
const {
S3Client,
GetObjectCommand,
HeadBucketCommand,
DeleteObjectCommand,
CreateBucketCommand
} = require('@aws-sdk/client-s3')
const { Upload } = require('@aws-sdk/lib-storage')
let s3Config = null
const getS3Config = () => {
if (!s3Config) {
if (!process.env.S3_ACCESS_KEY)
throw new Error('Config value S3_ACCESS_KEY is missing')
if (!process.env.S3_SECRET_KEY)
throw new Error('Config value S3_SECRET_KEY is missing')
if (!process.env.S3_ENDPOINT) throw new Error('Config value S3_ENDPOINT is missing')
s3Config = {
credentials: {
accessKeyId: process.env.S3_ACCESS_KEY,
secretAccessKey: process.env.S3_SECRET_KEY
},
endpoint: process.env.S3_ENDPOINT,
forcePathStyle: true,
// s3ForcePathStyle: true,
// signatureVersion: 'v4',
region: 'us-east-1'
}
}
return s3Config
}
let storageBucket = null
const getStorageBucket = () => {
if (!storageBucket) {
if (!process.env.S3_BUCKET) throw new Error('Config value S3_BUCKET is missing')
storageBucket = process.env.S3_BUCKET
}
return storageBucket
}
const getObjectStorage = () => ({
client: new S3Client(getS3Config()),
Bucket: getStorageBucket(),
createBucket: process.env.S3_CREATE_BUCKET || false
})
const getObjectStream = async ({ objectKey }) => {
const { client, Bucket } = getObjectStorage()
const data = await client.send(new GetObjectCommand({ Bucket, Key: objectKey }))
return data.Body
}
const getObjectAttributes = async ({ objectKey }) => {
const { client, Bucket } = getObjectStorage()
const data = await client.send(new GetObjectCommand({ Bucket, Key: objectKey }))
return { fileSize: data.ContentLength }
}
const storeFileStream = async ({ objectKey, fileStream }) => {
const { client, Bucket } = getObjectStorage()
const parallelUploads3 = new Upload({
client,
params: { Bucket, Key: objectKey, Body: fileStream },
tags: [
/*...*/
], // optional tags
queueSize: 4, // optional concurrency configuration
partSize: 1024 * 1024 * 5, // optional size of each part, in bytes, at least 5MB
leavePartsOnError: false // optional manually handle dropped parts
})
// parallelUploads3.on('httpUploadProgress', (progress) => {
// console.log(progress)
// })
const data = await parallelUploads3.done()
// the ETag is a hash of the object. Could be used to dedupe stuff...
return { fileHash: data.ETag }
}
const deleteObject = async ({ objectKey }) => {
const { client, Bucket } = getObjectStorage()
await client.send(new DeleteObjectCommand({ Bucket, Key: objectKey }))
}
const ensureStorageAccess = async () => {
const { client, Bucket, createBucket } = getObjectStorage()
try {
// await this._client.send(new HeadBucketCommand({ Bucket: this._bucket }))
await client.send(new HeadBucketCommand({ Bucket }))
return
} catch (err) {
if (err.statusCode === 403) {
throw new Error('Access denied to S3 bucket ')
}
if (createBucket) {
try {
const res = await client.send(new CreateBucketCommand({ Bucket }))
console.log(res)
} catch (err) {
console.log(err)
}
} else {
throw new Error(`Can't open S3 bucket '${Bucket}': ${err.toString()}`)
}
}
}
module.exports = {
ensureStorageAccess,
deleteObject,
getObjectAttributes,
storeFileStream,
getObjectStream
}
packages/server/modules/blobstorage/services.js
+107
View File
@@ -0,0 +1,107 @@
const knex = require('@/db/knex')
const { NotFoundError, ResourceMismatch } = require('@/modules/shared/errors')
const BlobStorage = () => knex('blob_storage')
const blobLookup = ({ blobId }) => BlobStorage().where({ id: blobId })
const uploadFileStream = async (
storeFileStream,
{ streamId, userId },
{ blobId, fileName, fileType, fileStream }
) => {
const objectKey = `assets/${streamId}/${blobId}`
const dbFile = {
id: blobId,
streamId,
userId,
objectKey,
fileName,
fileType
}
// need to insert the upload data before starting otherwise the upload finished
// even might fire faster, than the db insert, causing missing asset data in the db
await BlobStorage().insert(dbFile)
const { fileHash } = await storeFileStream({ objectKey, fileStream })
return { blobId, fileName, fileHash }
}
const getBlobMetadata = async ({ streamId, blobId }) => {
const obj = (await blobLookup({ blobId }).first()) || null
if (!obj) throw new NotFoundError(`The requested asset: ${blobId} doesn't exist`)
if (!streamId) throw new ResourceMismatch('No steamId provided')
if (obj.streamId !== streamId)
throw new ResourceMismatch("The stream doesn't have the given resource")
return obj
}
const blobQuery = ({ streamId, query }) => {
let blobs = BlobStorage().where({ streamId })
if (query) blobs = blobs.andWhereLike('fileName', `%${query}%`)
return blobs
}
const getBlobMetadataCollection = async ({ streamId, query, limit, cursor }) => {
const cursorTarget = 'createdAt'
const limitMax = 25
const queryLimit = limit && limit < limitMax ? limit : limitMax
const blobs = blobQuery({ streamId, query })
.orderBy(cursorTarget, 'desc')
.limit(queryLimit)
if (cursor) query.andWhere(cursorTarget, '<', cursor)
const rows = await blobs
return {
blobs: rows,
cursor: rows.length > 0 ? rows[rows.length - 1][cursorTarget].toISOString() : null
}
}
const blobCollectionSummary = async ({ streamId, query }) => {
const [summary] = await blobQuery({ streamId, query }).sum('fileSize').count('id')
return { totalSize: summary.sum ?? 0, totalCount: summary.count }
}
const getFileStream = async ({ getObjectStream, streamId, blobId }) => {
const { objectKey } = await getBlobMetadata({ streamId, blobId })
return await getObjectStream({ objectKey })
}
const markUploadSuccess = async (getObjectAttributes, streamId, blobId) =>
await updateBlobMetadata(streamId, blobId, async ({ objectKey }) => {
const { fileSize } = await getObjectAttributes({ objectKey })
return { uploadStatus: 1, fileSize }
})
const markUploadOverFileSizeLimit = async (deleteObject, streamId, blobId) =>
await markUploadError(deleteObject, streamId, blobId, 'File size limit reached')
const markUploadError = async (deleteObject, streamId, blobId, error) =>
await updateBlobMetadata(streamId, blobId, async ({ objectKey }) => {
await deleteObject({ objectKey })
return { uploadStatus: 2, uploadError: error }
})
const deleteBlob = async ({ streamId, blobId, deleteObject }) => {
const { objectKey } = await getBlobMetadata({ streamId, blobId })
await deleteObject({ objectKey })
await blobLookup({ blobId }).del()
}
const updateBlobMetadata = async (streamId, blobId, updateCallback) => {
const { objectKey, fileName } = await getBlobMetadata({ streamId, blobId })
const updateData = await updateCallback({ objectKey })
await blobLookup({ blobId }).update(updateData)
return { blobId, fileName, ...updateData }
}
module.exports = {
getBlobMetadata,
uploadFileStream,
markUploadSuccess,
markUploadOverFileSizeLimit,
markUploadError,
getFileStream,
deleteBlob,
getBlobMetadataCollection,
blobCollectionSummary
}
packages/server/modules/blobstorage/tests/unit/blobstorage.spec.js
View File
packages/server/modules/comments/graph/resolvers/comments.js
+23 -19
View File
@@ -1,6 +1,10 @@
const { pubsub } = require('@/modules/shared')
const { ForbiddenError, ApolloError, withFilter } = require('apollo-server-express')
const { Forbidden } = require('@/modules/shared/errors')
const {
ForbiddenError: ApolloForbiddenError,
ApolloError,
withFilter
} = require('apollo-server-express')
const { ForbiddenError } = require('@/modules/shared/errors')
const { getStream } = require('@/modules/core/services/streams')
const { Roles } = require('@/modules/core/helpers/mainConstants')
const { saveActivity } = require('@/modules/activitystream/services')
@@ -29,7 +33,7 @@ const authorizeStreamAccess = async ({
requireRole = false
}) => {
if (serverRole === Roles.Server.ArchivedUser)
throw new ForbiddenError('You are not authorized.')
throw new ApolloForbiddenError('You are not authorized.')
const stream = await getStream({ streamId, userId })
if (!stream) throw new ApolloError('Stream not found')
@@ -42,7 +46,7 @@ const authorizeStreamAccess = async ({
if (stream.isPublic && requireRole && !stream.allowPublicComments && !stream.role)
authZed = false
if (!authZed) throw new ForbiddenError('You are not authorized.')
if (!authZed) throw new ApolloForbiddenError('You are not authorized.')
return stream
}
@@ -57,7 +61,7 @@ module.exports = {
})
const comment = await getComment({ id: args.id, userId: context.userId })
if (comment.streamId !== args.streamId)
throw new ForbiddenError('You do not have access to this comment.')
throw new ApolloForbiddenError('You do not have access to this comment.')
return comment
},
@@ -92,14 +96,14 @@ module.exports = {
Stream: {
async commentCount(parent, args, context) {
if (context.role === Roles.Server.ArchivedUser)
throw new ForbiddenError('You are not authorized.')
throw new ApolloForbiddenError('You are not authorized.')
return await getStreamCommentCount({ streamId: parent.id })
}
},
Commit: {
async commentCount(parent, args, context) {
if (context.role === Roles.Server.ArchivedUser)
throw new ForbiddenError('You are not authorized.')
throw new ApolloForbiddenError('You are not authorized.')
return await getResourceCommentCount({ resourceId: parent.id })
}
},
@@ -107,14 +111,14 @@ module.exports = {
// urgh, i think we tripped our gql schemas in there a bit
async commentCount(parent, args, context) {
if (context.role === Roles.Server.ArchivedUser)
throw new ForbiddenError('You are not authorized.')
throw new ApolloForbiddenError('You are not authorized.')
return await getResourceCommentCount({ resourceId: parent.id })
}
},
Object: {
async commentCount(parent, args, context) {
if (context.role === Roles.Server.ArchivedUser)
throw new ForbiddenError('You are not authorized.')
throw new ApolloForbiddenError('You are not authorized.')
return await getResourceCommentCount({ resourceId: parent.id })
}
},
@@ -155,7 +159,7 @@ module.exports = {
})
if (!stream.allowPublicComments && !stream.role)
throw new ForbiddenError('You are not authorized.')
throw new ApolloForbiddenError('You are not authorized.')
await pubsub.publish('COMMENT_THREAD_ACTIVITY', {
commentThreadActivity: { eventType: 'reply-typing-status', data: args.data },
@@ -167,7 +171,7 @@ module.exports = {
async commentCreate(parent, args, context) {
if (!context.userId)
throw new ForbiddenError('Only registered users can comment.')
throw new ApolloForbiddenError('Only registered users can comment.')
const stream = await getStream({
streamId: args.input.streamId,
@@ -175,7 +179,7 @@ module.exports = {
})
if (!stream.allowPublicComments && !stream.role)
throw new ForbiddenError('You are not authorized.')
throw new ApolloForbiddenError('You are not authorized.')
const { id, text } = await createComment({
userId: context.userId,
@@ -226,7 +230,7 @@ module.exports = {
await editComment({ userId: context.userId, input: args.input, matchUser })
return true
} catch (err) {
if (err instanceof Forbidden) throw new ForbiddenError(err.message)
if (err instanceof ForbiddenError) throw new ApolloForbiddenError(err.message)
throw err
}
},
@@ -256,7 +260,7 @@ module.exports = {
try {
await archiveComment({ ...args, userId: context.userId }) // NOTE: permissions check inside service
} catch (err) {
if (err instanceof Forbidden) throw new ForbiddenError(err.message)
if (err instanceof ForbiddenError) throw new ApolloForbiddenError(err.message)
throw err
}
@@ -282,7 +286,7 @@ module.exports = {
async commentReply(parent, args, context) {
if (!context.userId)
throw new ForbiddenError('Only registered users can comment.')
throw new ApolloForbiddenError('Only registered users can comment.')
const stream = await getStream({
streamId: args.input.streamId,
@@ -290,7 +294,7 @@ module.exports = {
})
if (!stream.allowPublicComments && !stream.role)
throw new ForbiddenError('You are not authorized.')
throw new ApolloForbiddenError('You are not authorized.')
const { id, text } = await createCommentReply({
authorId: context.userId,
@@ -337,7 +341,7 @@ module.exports = {
})
if (!stream.allowPublicComments && !stream.role)
throw new ForbiddenError('You are not authorized.')
throw new ApolloForbiddenError('You are not authorized.')
return (
payload.streamId === variables.streamId &&
@@ -356,7 +360,7 @@ module.exports = {
})
if (!stream.allowPublicComments && !stream.role)
throw new ForbiddenError('You are not authorized.')
throw new ApolloForbiddenError('You are not authorized.')
// if we're listening for a stream's root comments events
if (!variables.resourceIds) {
@@ -399,7 +403,7 @@ module.exports = {
})
if (!stream.allowPublicComments && !stream.role)
throw new ForbiddenError('You are not authorized.')
throw new ApolloForbiddenError('You are not authorized.')
return (
payload.streamId === variables.streamId &&
packages/server/modules/comments/services/commentTextService.js
+1 -1
View File
@@ -1,4 +1,4 @@
const { RichTextParseError } = require('@/modules/core/errors/base')
const { RichTextParseError } = require('@/modules/shared/errors')
const {
isTextEditorValueSchema,
isTextEditorDoc,
packages/server/modules/comments/services/index.js
+3 -3
View File
@@ -1,7 +1,7 @@
'use strict'
const crs = require('crypto-random-string')
const knex = require('@/db/knex')
const { Forbidden } = require('@/modules/shared/errors')
const { ForbiddenError } = require('@/modules/shared/errors')
const {
buildCommentTextFromInput
} = require('@/modules/comments/services/commentTextService')
@@ -134,7 +134,7 @@ module.exports = {
const editedComment = await Comments().where({ id: input.id }).first()
if (!editedComment) throw new Error("The comment doesn't exist")
if (matchUser && editedComment.authorId !== userId)
throw new Forbidden("You cannot edit someone else's comments")
throw new ForbiddenError("You cannot edit someone else's comments")
const newText = buildCommentTextFromInput(input.text)
await Comments().where({ id: input.id }).update({ text: newText })
@@ -181,7 +181,7 @@ module.exports = {
if (comment.authorId !== userId) {
if (!aclEntry || aclEntry.role !== 'stream:owner')
throw new Forbidden("You don't have permission to archive the comment")
throw new ForbiddenError("You don't have permission to archive the comment")
}
await Comments().where({ id: commentId }).update({ archived })
packages/server/modules/core/repositories/streams.js
+1 -1
View File
@@ -1,6 +1,6 @@
const _ = require('lodash')
const { Streams, StreamAcl, StreamFavorites, knex } = require('@/modules/core/dbSchema')
const { InvalidArgumentError } = require('@/modules/core/errors/base')
const { InvalidArgumentError } = require('@/modules/shared/errors')
const { Roles } = require('@/modules/core/helpers/mainConstants')
/**
packages/server/modules/core/services/streams.js
+4 -10
View File
@@ -13,10 +13,7 @@ const {
setStreamFavorited,
canUserFavoriteStream
} = require('@/modules/core/repositories/streams')
const {
UnauthorizedAccessError,
InvalidArgumentError
} = require('@/modules/core/errors/base')
const { UnauthorizedError, InvalidArgumentError } = require('@/modules/shared/errors')
/**
* Get base query for finding or counting user streams
@@ -288,12 +285,9 @@ module.exports = {
async favoriteStream({ userId, streamId, favorited }) {
// Check if user has access to stream
if (!(await canUserFavoriteStream({ userId, streamId }))) {
throw new UnauthorizedAccessError(
"User doesn't have access to the specified stream",
{
info: { userId, streamId }
}
)
throw new UnauthorizedError("User doesn't have access to the specified stream", {
info: { userId, streamId }
})
}
// Favorite/unfavorite the stream
packages/server/modules/fileuploads/index.js
+43 -142
View File
@@ -2,22 +2,29 @@
'use strict'
const debug = require('debug')
const Busboy = require('busboy')
const { contextMiddleware } = require('@/modules/shared')
const { saveUploadFile } = require('./services/fileuploads')
const request = require('request')
const {
contextMiddleware,
validateScopes,
authorizeResolver
} = require('@/modules/shared')
authMiddlewareCreator,
streamWritePermissions
} = require('@/modules/shared/authz')
const {
checkBucket,
startUploadFile,
finishUploadFile,
getFileInfo,
getFileStream
} = require('./services/fileuploads')
const { getStream } = require('../core/services/streams')
const saveFileUploads = async ({ userId, streamId, branchName, uploadResults }) => {
await Promise.all(
uploadResults.map(async (upload) => {
await saveUploadFile({
fileId: upload.blobId,
streamId,
branchName,
userId,
fileName: upload.fileName,
fileType: upload.fileName.split('.').pop(),
fileSize: upload.fileSize
})
})
)
}
exports.init = async (app) => {
if (process.env.DISABLE_FILE_UPLOADS) {
@@ -27,136 +34,30 @@ exports.init = async (app) => {
debug('speckle:modules')('📄 Init FileUploads module')
}
if (!process.env.S3_BUCKET) {
debug('speckle:modules')(
'ERROR: S3_BUCKET env variable was not specified. File uploads will be DISABLED.'
)
return
}
await checkBucket()
const checkStreamPermissions = async (req) => {
if (!req.context || !req.context.auth) {
return { hasPermissions: false, httpErrorCode: 401 }
}
try {
await validateScopes(req.context.scopes, 'streams:write')
} catch (err) {
return { hasPermissions: false, httpErrorCode: 401 }
}
try {
await authorizeResolver(
req.context.userId,
req.params.streamId,
'stream:contributor'
app.post(
'/api/file/:fileType/:streamId/:branchName?',
contextMiddleware,
authMiddlewareCreator(streamWritePermissions),
async (req, res) => {
req.pipe(
request(
`${process.env.CANONICAL_URL}/api/stream/${req.params.streamId}/blob`,
async (err, response, body) => {
if (response.statusCode === 201) {
const { uploadResults } = JSON.parse(body)
await saveFileUploads({
userId: req.context.userId,
streamId: req.params.streamId,
branchName: req.params.branchName ?? 'main',
uploadResults
})
}
res.status(response.statusCode).send(body)
}
)
)
} catch (err) {
return { hasPermissions: false, httpErrorCode: 401 }
}
return { hasPermissions: true, httpErrorCode: 200 }
}
app.get('/api/file/:fileId', contextMiddleware, async (req, res) => {
if (process.env.DISABLE_FILE_UPLOADS) {
return res.status(503).send('File uploads are disabled on this server')
}
const fileInfo = await getFileInfo({ fileId: req.params.fileId })
if (!fileInfo) return res.status(404).send('File not found')
// Check stream read access
const streamId = fileInfo.streamId
const stream = await getStream({ streamId, userId: req.context.userId })
if (!stream) {
return res.status(404).send('File stream not found')
}
if (!stream.isPublic && req.context.auth === false) {
return res.status(401).send('You must be logged in to access private streams')
}
if (!stream.isPublic) {
try {
await validateScopes(req.context.scopes, 'streams:read')
} catch (err) {
return res.status(401).send("The provided auth token can't read streams")
}
try {
await authorizeResolver(req.context.userId, streamId, 'stream:reviewer')
} catch (err) {
return res.status(401).send("You don't have access to this private stream")
}
}
const fileStream = await getFileStream({ fileId: req.params.fileId })
res.writeHead(200, {
'Content-Type': 'application/octet-stream',
'Content-Disposition': `attachment; filename="${fileInfo.fileName}"`
})
fileStream.pipe(res)
}),
app.post(
'/api/file/:fileType/:streamId/:branchName?',
contextMiddleware,
async (req, res) => {
if (process.env.DISABLE_FILE_UPLOADS) {
return res.status(503).send('File uploads are disabled on this server')
}
const { hasPermissions, httpErrorCode } = await checkStreamPermissions(req)
if (!hasPermissions) {
return res.status(httpErrorCode).end()
}
const fileUploadPromises = []
const busboy = Busboy({ headers: req.headers })
busboy.on('file', (name, file, info) => {
const { filename } = info
let fileType = req.params.fileType
if (fileType === 'autodetect')
fileType = filename.split('.').pop().toLowerCase()
const promise = startUploadFile({
streamId: req.params.streamId,
branchName: req.params.branchName || '',
userId: req.context.userId,
fileName: filename,
fileType,
fileStream: file
})
fileUploadPromises.push(promise)
})
busboy.on('finish', async function () {
const fileIds = []
for (const promise of fileUploadPromises) {
const fileId = await promise
fileIds.push(fileId)
}
for (const fileId of fileIds) {
await finishUploadFile({ fileId })
}
res.send(fileIds)
})
busboy.on('error', async (err) => {
console.log(`FileUpload error: ${err}`)
res.status(400).end('Upload request error. The server logs have more details')
})
req.pipe(busboy)
}
)
)
}
exports.finalize = () => {}
packages/server/modules/fileuploads/services/fileuploads.js
+6 -68
View File
@@ -1,42 +1,11 @@
/* istanbul ignore file */
'use strict'
const crs = require('crypto-random-string')
const knex = require('@/db/knex')
const S3 = require('aws-sdk/clients/s3')
const FileUploads = () => knex('file_uploads')
function getS3Config() {
return {
accessKeyId: process.env.S3_ACCESS_KEY || 'minioadmin',
secretAccessKey: process.env.S3_SECRET_KEY || 'minioadmin',
endpoint: process.env.S3_ENDPOINT || 'http://127.0.0.1:9000',
s3ForcePathStyle: true,
signatureVersion: 'v4'
}
}
module.exports = {
async checkBucket() {
const s3 = new S3(getS3Config())
const Bucket = process.env.S3_BUCKET
try {
await s3.headBucket({ Bucket }).promise()
return
} catch (err) {
if (err.statusCode === 403) {
throw new Error('Access denied to S3 bucket ')
}
if (process.env.S3_CREATE_BUCKET === 'true') {
await s3.createBucket({ Bucket }).promise()
} else {
throw new Error(`Can't open S3 bucket '${Bucket}': ${err.toString()}`)
}
}
},
async getFileInfo({ fileId }) {
const fileInfo = await FileUploads().where({ id: fileId }).select('*').first()
return fileInfo
@@ -50,56 +19,25 @@ module.exports = {
return fileInfos
},
async getFileStream({ fileId }) {
const s3 = new S3(getS3Config())
const Bucket = process.env.S3_BUCKET
const Key = `files/${fileId}`
const fileStream = s3.getObject({ Key, Bucket }).createReadStream()
return fileStream
},
async startUploadFile({
async saveUploadFile({
fileId,
streamId,
branchName,
userId,
fileName,
fileType,
fileStream
fileSize
}) {
// Create ID and db entry
const fileId = crs({ length: 10 })
const dbFile = {
id: fileId,
streamId,
branchName,
userId,
fileName,
fileType
fileType,
fileSize,
uploadComplete: true
}
await FileUploads().insert(dbFile)
// Upload stream
const s3 = new S3(getS3Config())
const Bucket = process.env.S3_BUCKET
// TODO: error if missing
const Key = `files/${fileId}`
await s3.upload({ Bucket, Key, Body: fileStream }).promise()
return fileId
},
async finishUploadFile({ fileId }) {
const s3 = new S3(getS3Config())
const Bucket = process.env.S3_BUCKET
// TODO: error if missing
const Key = `files/${fileId}`
// Get file size and update db entry
const headResponse = await s3.headObject({ Key, Bucket }).promise()
const fileSize = headResponse.ContentLength
await FileUploads().where({ id: fileId }).update({ uploadComplete: true, fileSize })
}
}
packages/server/modules/index.js
+2 -1
View File
@@ -16,7 +16,8 @@ exports.init = async (app) => {
'./serverinvites',
'./previews',
'./fileuploads',
'./comments'
'./comments',
'./blobstorage'
]
// Stage 1: initialise all modules
packages/server/modules/shared/authz.js
+196
View File
@@ -0,0 +1,196 @@
const { Roles, Scopes } = require('@/modules/core/helpers/mainConstants')
const { getStream } = require('@/modules/core/services/streams')
const { getRoles } = require('@/modules/shared')
const {
ForbiddenError: SFE,
UnauthorizedError: SUE,
ContextError
} = require('@/modules/shared/errors')
const authFailed = (context, error = null) => ({
context,
authResult: { authorized: false, error }
})
const authSuccess = (context) => ({
context,
authResult: { authorized: true, error: null }
})
const validateRole =
({ requiredRole, rolesLookup, iddqd, roleGetter }) =>
async ({ context, authResult }) => {
const roles = await rolesLookup()
// having the required role doesn't rescue from authResult failure
if (authResult.error) return { context, authResult }
// role validation has nothing to do with auth...
//this check doesn't belong here, move it out to the auth pipeline
if (!context.auth)
return authFailed(context, new SUE('Cannot validate role without auth'))
const role = roles.find((r) => r.name === requiredRole)
const myRole = roles.find((r) => r.name === roleGetter(context))
if (!role) return authFailed(context, new SFE('Invalid role requirement specified'))
if (!myRole) return authFailed(context, new SFE('Your role is not valid'))
if (myRole.name === iddqd || myRole.weight >= role.weight)
return authSuccess(context)
return authFailed(context, new SFE('You do not have the required role'))
}
const validateServerRole = ({ requiredRole }) =>
validateRole({
requiredRole,
rolesLookup: getRoles,
iddqd: Roles.Server.Admin,
roleGetter: (context) => context.role
})
const validateStreamRole = ({ requiredRole }) =>
validateRole({
requiredRole,
rolesLookup: getRoles,
iddqd: Roles.Stream.Owner,
roleGetter: (context) => context.stream.role
})
// this could be still useful, if the operation doesnt require a stream context
// const authorizeResolver = refactor the implementation in ../index.js
const validateScope =
({ requiredScope }) =>
async ({ context, authResult }) => {
// having the required role doesn't rescue from authResult failure
if (authResult.error) return { context, authResult }
if (!context.scopes)
return authFailed(context, new SFE('You do not have the required privileges.'))
if (
context.scopes.indexOf(requiredScope) === -1 &&
context.scopes.indexOf('*') === -1
)
return authFailed(context, new SFE('You do not have the required privileges.'))
return authSuccess(context)
}
// this doesn't do any checks on the scopes, its sole responsibility is to add the
// stream object to the pipeline context
const contextRequiresStream =
(streamGetter) =>
// stream getter is an async func over { streamId, userId } returning a stream object
// IoC baby...
async ({ context, authResult, params }) => {
if (!params?.streamId)
return authFailed(
context,
new ContextError("The context doesn't have a streamId")
)
// because we're assigning to the context, it would raise if it would be null
// its probably?? safer than returning a new context
if (!context)
return authFailed(context, new ContextError('The context is not defined'))
// cause stream getter could throw, its not a safe function if we want to
// keep the pipeline rolling
try {
const stream = await streamGetter({
streamId: params.streamId,
userId: context?.userId
})
context.stream = stream
return { context, authResult }
} catch (err) {
// this prob needs some more detailing to not leak internal errors
return authFailed(context, new ContextError(err.message))
}
}
const allowForRegisteredUsersOnPublicStreamsEvenWithoutRole = async ({
context,
authResult
}) =>
context.auth && context.stream.isPublic
? authSuccess(context)
: { context, authResult }
const allowForAllRegisteredUsersOnPublicStreamsWithPublicComments = async ({
context,
authResult
}) =>
context.auth && context.stream.isPublic && context.stream.allowPublicComments
? authSuccess(context)
: { context, authResult }
const authPipelineCreator = (steps) => {
const pipeline = async ({ context, params }) => {
let authResult = { authorized: false, error: null }
for (const step of steps) {
;({ context, authResult } = await step({ context, authResult, params }))
}
// validate auth result a bit...
if (authResult.authorized && authResult.error)
throw new Error('a big fuckup on our end')
return { context, authResult }
}
return pipeline
}
//we could even add an auth middleware creator
// todo move this to a webserver related module, it has no place here
const authMiddlewareCreator = (steps) => {
const pipeline = authPipelineCreator(steps)
const middleware = async (req, res, next) => {
const { authResult } = await pipeline({ context: req.context, params: req.params })
if (!authResult.authorized) {
let message = 'Unknown AuthZ error'
let status = 500
if (authResult.error) {
message = authResult.error.message
if (authResult.error instanceof SUE) status = 401
if (authResult.error instanceof SFE) status = 403
}
return res.status(status).send(message)
}
next()
}
return middleware
}
// eslint-disable-next-line no-unused-vars
const exampleMiddleware = authMiddlewareCreator([
// at some point add the context preparation here too
validateServerRole({ requiredRole: Roles.Server.User }),
validateScope({ requiredScope: Scopes.Streams.Write }),
contextRequiresStream(getStream),
validateStreamRole({ requiredRole: Roles.Stream.Reviewer }),
allowForRegisteredUsersOnPublicStreamsEvenWithoutRole
])
module.exports = {
authPipelineCreator,
authSuccess,
authFailed,
validateRole,
validateScope,
validateServerRole,
validateStreamRole,
contextRequiresStream,
ContextError,
authMiddlewareCreator,
allowForRegisteredUsersOnPublicStreamsEvenWithoutRole,
allowForAllRegisteredUsersOnPublicStreamsWithPublicComments,
streamWritePermissions: [
validateServerRole({ requiredRole: Roles.Server.User }),
validateScope({ requiredScope: Scopes.Streams.Write }),
contextRequiresStream(getStream),
validateStreamRole({ requiredRole: Roles.Stream.Contributor })
],
streamReadPermissions: [
validateServerRole({ requiredRole: Roles.Server.User }),
validateScope({ requiredScope: Scopes.Streams.Read }),
contextRequiresStream(getStream),
validateStreamRole({ requiredRole: Roles.Stream.Contributor })
]
}
packages/server/modules/shared/errors.js
-8
View File
@@ -1,8 +0,0 @@
class Forbidden extends Error {
constructor(message) {
super(message)
this.name = 'Forbidden'
}
}
module.exports = { Forbidden }
packages/server/modules/core/errors/base.js → packages/server/modules/shared/errors/base.js
+1 -28
View File
@@ -57,31 +57,4 @@ class BaseError extends VError {
}
}
/**
* Use this to validate args
*/
class InvalidArgumentError extends BaseError {
static code = 'INVALID_ARGUMENT_ERROR'
static defaultMessage = 'Invalid arguments received'
}
/**
* Use this to throw when user tries to access data that he shouldn't have access to
*/
class UnauthorizedAccessError extends BaseError {
static code = 'UNAUTHORIZED_ACCESS_ERROR'
static defaultMessage = 'Attempted unauthorized access to data'
}
class RichTextParseError extends BaseError {
static code = 'RICH_TEXT_PARSE_ERROR'
static defaultMessage =
'An error occurred while trying to parse the rich text document'
}
module.exports = {
BaseError,
InvalidArgumentError,
UnauthorizedAccessError,
RichTextParseError
}
module.exports = { BaseError }
packages/server/modules/shared/errors/index.js
+51
View File
@@ -0,0 +1,51 @@
const { BaseError } = require('./base')
class ForbiddenError extends BaseError {
static code = 'FORBIDDEN_ERROR'
static defaultMessage = 'Access to the resource is forbidden'
}
/**
* Use this to throw when user tries to access data that he shouldn't have access to
*/
class UnauthorizedError extends BaseError {
static code = 'UNAUTHORIZED_ACCESS_ERROR'
static defaultMessage = 'Attempted unauthorized access to data'
}
class NotFoundError extends BaseError {
static code = 'NOT_FOUND_ERROR'
static defaultMessage = "These aren't the droids you're looking for."
}
class ResourceMismatch extends BaseError {
static code = 'BAD_REQUEST_ERROR'
static defaultMessage = 'The target resources mismatch'
}
/**
* Use this to validate args
*/
class InvalidArgumentError extends BaseError {
static code = 'INVALID_ARGUMENT_ERROR'
static defaultMessage = 'Invalid arguments received'
}
class RichTextParseError extends BaseError {
static code = 'RICH_TEXT_PARSE_ERROR'
static defaultMessage =
'An error occurred while trying to parse the rich text document'
}
class ContextError extends BaseError {
static code = 'CONTEXT_ERROR'
static defaultMessage = 'The context is missing from the request'
}
module.exports = {
ForbiddenError,
UnauthorizedError,
NotFoundError,
ResourceMismatch,
InvalidArgumentError,
RichTextParseError,
ContextError
}
packages/server/modules/shared/index.js
+10 -3
View File
@@ -48,7 +48,7 @@ async function buildContext({ req, connection }) {
}
/**
* Graphql server context helper: sets req.context to have an auth prop (true/false), userId and server role.
* Not just Graphql server context helper: sets req.context to have an auth prop (true/false), userId and server role.
* @returns {AuthContextPart}
*/
async function contextApiTokenHelper({ req, connection }) {
@@ -92,6 +92,12 @@ async function contextMiddleware(req, res, next) {
let roles
const getRoles = async () => {
if (roles) return roles
roles = await knex('user_roles').select('*')
return roles
}
/**
* Validates a server role against the req's context object.
* @param {[type]} context [description]
@@ -99,7 +105,7 @@ let roles
* @return {[type]} [description]
*/
async function validateServerRole(context, requiredRole) {
if (!roles) roles = await knex('user_roles').select('*')
const roles = await getRoles()
if (!context.auth) throw new ForbiddenError('You must provide an auth token.')
@@ -202,5 +208,6 @@ module.exports = {
validateServerRole,
validateScopes,
authorizeResolver,
pubsub
pubsub,
getRoles
}
packages/server/modules/shared/test/authz.spec.js
+219
View File
@@ -0,0 +1,219 @@
const expect = require('chai').expect
const {
authPipelineCreator,
authFailed,
authSuccess,
validateRole,
validateScope,
contextRequiresStream,
ContextError
} = require('@/modules/shared/authz')
const {
ForbiddenError: SFE,
UnauthorizedError: SUE
} = require('@/modules/shared/errors')
describe('AuthZ @shared', () => {
it('Empty pipeline returns no authorization', async () => {
const pipeline = authPipelineCreator([])
const { authResult } = await pipeline({ context: { foo: 'bar' } })
expect(authResult.authorized).to.equal(false)
})
describe('Role validation', () => {
const rolesLookup = async () => [
{ name: '1', weight: 1 },
{ name: '2', weight: 2 },
{ name: '3', weight: 3 },
{ name: 'goku', weight: 9001 },
{ name: '42', weight: 42 }
]
const testData = [
{
name: 'Having lower privileged role than required results auth failed',
requiredRole: '2',
context: { auth: true, role: '1' },
expectedResult: authFailed(null, new SFE('You do not have the required role'))
},
{
name: 'Not having auth fails role validation',
requiredRole: '2',
context: { auth: false },
expectedResult: authFailed(null, new SUE('Cannot validate role without auth'))
},
{
name: 'Requiring a junk role fails auth',
requiredRole: 'knock knock...',
context: { auth: true, role: '1' },
expectedResult: authFailed(null, new SFE('Invalid role requirement specified'))
},
{
name: 'Having a junk role fails auth',
requiredRole: '2',
context: { auth: true, role: 'iddqd' },
expectedResult: authFailed(null, new SFE('Your role is not valid'))
},
{
name: 'Not having the required level fails',
requiredRole: 'goku',
context: { auth: true, role: '3' },
expectedResult: authFailed(null, new SFE('You do not have the required role'))
},
{
name: 'Having the god mode role defeats even higher privilege requirement',
requiredRole: 'goku',
context: { auth: true, role: '42' },
expectedResult: authSuccess()
},
{
name: 'Having equal role weight to required succeeds',
requiredRole: '3',
context: { auth: true, role: '3' },
expectedResult: authSuccess()
},
{
name: 'Having bigger role weight than required succeeds',
requiredRole: '3',
context: { auth: true, role: 'goku' },
expectedResult: authSuccess()
}
]
testData.forEach((testCase) =>
it(`${testCase.name}`, async () => {
const step = validateRole({
requiredRole: testCase.requiredRole,
rolesLookup,
iddqd: '42',
roleGetter: (context) => context.role
})
const { authResult, context } = await step({
context: testCase.context,
authResult: authFailed()
})
expect(authResult.authorized).to.exist
expect(authResult.authorized).to.equal(
testCase.expectedResult.authResult.authorized
)
// this also needs to check for the error type... is this how do you do that in JS????
expect(authResult.error?.name).to.equal(
testCase.expectedResult.authResult.error?.name
)
expect(authResult.error?.message).to.equal(
testCase.expectedResult.authResult.error?.message
)
expect(context).to.deep.equal(testCase.context)
})
)
it('Role validation fails if input authResult is already in an error state', async () => {
const step = validateRole({ requiredRole: 'goku', rolesLookup, iddqd: '42' })
const error = new SFE('This will be echoed back')
const { authResult } = await step({
context: {},
authResult: { authorized: false, error }
})
expect(authResult.authorized).to.be.false
expect(authResult.error.message).to.equal(error.message)
expect(authResult.error.name).to.equal(error.name)
})
})
describe('Validate scopes', () => {
it('Scope validation fails if input authResult is already in an error state', async () => {
const step = validateScope({ requiredScope: 'play mahjong' })
const expectedError = new SFE("Scope validation doesn't rescue the auth pipeline")
const { authResult } = await step({
context: {},
authResult: { authorized: false, error: expectedError }
})
expect(authResult.authorized).to.be.false
expect(authResult.error.message).to.equal(expectedError.message)
expect(authResult.error.name).to.equal(expectedError.name)
})
it('Without having any scopes on the context cannot validate scopes', async () => {
const step = validateScope({ requiredScope: 'play mahjong' })
const { authResult } = await step({ context: {}, authResult: {} })
expect(authResult.authorized).to.equal(false)
const expectedError = new SFE('You do not have the required privileges.')
expect(authResult.error.message).to.equal(expectedError.message)
expect(authResult.error.name).to.equal(expectedError.name)
})
it('Not having the right scopes results auth failed', async () => {
const step = validateScope({ requiredScope: 'play mahjong' })
const { authResult } = await step({
context: { scopes: ['sit around and wait', 'try to be cool'] },
authResult: {}
})
expect(authResult.authorized).to.equal(false)
const expectedError = new SFE('You do not have the required privileges.')
expect(authResult.error.message).to.equal(expectedError.message)
expect(authResult.error.name).to.equal(expectedError.name)
})
it('Having the right scopes results auth success', async () => {
const step = validateScope({ requiredScope: 'play mahjong' })
const { authResult } = await step({
context: { scopes: ['sit around and wait', 'try to be cool', 'play mahjong'] },
authResult: {}
})
expect(authResult.authorized).to.equal(true)
expect(authResult.error).to.not.exist
})
})
describe('Context requires stream', () => {
const expectAuthError = (expectedError, authResult) => {
expect(authResult.authorized).to.be.false
expect(authResult.error).to.exist
expect(authResult.error.message).to.equal(expectedError.message)
expect(authResult.error.name).to.equal(expectedError.name)
}
it('Without streamId in the params it raises context error', async () => {
const step = contextRequiresStream(async () => ({ ur: 'bamboozled' }))
const { authResult } = await step({ params: {} })
expectAuthError(
new ContextError("The context doesn't have a streamId"),
authResult
)
})
it('If params is not defined it raises context error', async () => {
const step = contextRequiresStream(async () => ({ ur: 'bamboozled' }))
const { authResult } = await step({})
expectAuthError(
new ContextError("The context doesn't have a streamId"),
authResult
)
})
it('Stream is added to the returned context object', async () => {
const demoStream = {
id: 'foo',
name: 'bar'
}
const step = contextRequiresStream(async () => demoStream)
const { context } = await step({
context: {},
params: { streamId: 'this is fake and its fine' }
})
expect(context.stream).to.deep.equal(demoStream)
})
it('If context is not defined return auth failure', async () => {
const step = contextRequiresStream(async () => {})
const { authResult } = await step({ params: { streamId: 'the need for stream' } })
expectAuthError(new ContextError('The context is not defined'), authResult)
})
it('If stream getter raises, the error is handled', async () => {
const errorMessage = 'oh dangit'
const step = contextRequiresStream(async () => {
throw new Error(errorMessage)
})
const { authResult } = await step({
context: {},
params: { streamId: 'the need for stream' }
})
expectAuthError(new ContextError(errorMessage), authResult)
})
})
})
packages/server/package.json
+4 -2
View File
@@ -24,15 +24,15 @@
"cli": "./bin/cli"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.100.0",
"@aws-sdk/lib-storage": "^3.100.0",
"@godaddy/terminus": "^4.9.0",
"@sentry/node": "^6.17.9",
"@sentry/tracing": "^6.17.9",
"apollo-server-express": "^2.19.0",
"apollo-server-testing": "^2.19.0",
"auto-load": "^3.0.4",
"aws-sdk": "^2.1075.0",
"bcrypt": "^5.0.0",
"body-parser": "^1.19.2",
"busboy": "^1.4.0",
"compression": "^1.7.4",
"connect-redis": "^6.1.1",
@@ -67,11 +67,13 @@
"pg-query-stream": "^4.2.3",
"prom-client": "^14.0.1",
"redis": "^3.1.1",
"request": "^2.88.2",
"response-time": "^2.3.2",
"sanitize-html": "^2.7.0",
"sharp": "^0.29.3",
"string-pixel-width": "^1.10.0",
"subscriptions-transport-ws": "0.9.0",
"undici": "^5.4.0",
"verror": "^1.10.1",
"xml-escape": "^1.1.0",
"zxcvbn": "^4.4.2"
packages/webhook-service/src/knex.js
+9 -3
View File
@@ -1,9 +1,15 @@
/* eslint-disable camelcase */
'use strict'
module.exports = require('knex')({
client: 'pg',
connection:
process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@localhost/speckle',
pool: { min: 1, max: 1 }
connection: {
application_name: 'speckle_webhook_service',
connectionString:
process.env.PG_CONNECTION_STRING ||
'postgres://speckle:speckle@localhost/speckle',
query_timeout: 4.32e7
},
pool: { min: 0, max: 1 }
// migrations are in managed in the server package
})
utils/helm/speckle-server/templates/deployment-backend.yml
+2
View File
@@ -107,6 +107,8 @@ spec:
secretKeyRef:
name: {{ .Values.secretName }}
key: postgres_url
- name: POSTGRES_MAX_CONNECTIONS_SERVER
value: {{ .Values.db.maxConnectionsServer | quote }}
- name: PGSSLMODE
value: "{{ .Values.db.PGSSLMODE }}"
utils/helm/speckle-server/values.yaml
+1
View File
@@ -8,6 +8,7 @@ docker_image_tag: v2.3.3
db:
# postgres_url: secret -> postgres_url
useCertificate: false
maxConnectionsServer: 4
certificate: '' # Multi-line string with the contents of `ca-certificate.crt`
PGSSLMODE: require
yarn.lock
+1135 -15
View File
File diff suppressed because it is too large Load Diff