'use strict' const crypto = require('crypto') const crs = require('crypto-random-string') const bcrypt = require('bcrypt') const { chunk } = require('lodash') const { logger: parentLogger } = require('../observability/logging') const knex = require('../knex') const Observability = require('@speckle/shared/dist/commonjs/observability/index.js') const Streams = () => knex('streams') const Branches = () => knex('branches') const Objects = () => knex('objects') const Closures = () => knex('object_children_closure') const ApiTokens = () => knex('api_tokens') const TokenScopes = () => knex('token_scopes') module.exports = class ServerAPI { constructor({ streamId, logger }) { this.streamId = streamId this.isSending = false this.buffer = [] this.logger = logger || Observability.extendLoggerComponent(parentLogger.child({ streamId }), 'ifc') } async saveObject(obj) { if (!obj) throw new Error('Null object') if (!obj.id) { obj.id = crypto.createHash('md5').update(JSON.stringify(obj)).digest('hex') } await this.createObject({ streamId: this.streamId, object: obj }) return obj.id } async saveObjectBatch(objs) { return await this.createObjectsBatched(this.streamId, objs) } async createObject({ streamId, object }) { const insertionObject = this.prepInsertionObject(streamId, object) const closures = [] const totalChildrenCountByDepth = {} if (object.__closure !== null) { for (const prop in object.__closure) { closures.push({ streamId, parent: insertionObject.id, child: prop, minDepth: object.__closure[prop] }) if (totalChildrenCountByDepth[object.__closure[prop].toString()]) totalChildrenCountByDepth[object.__closure[prop].toString()]++ else totalChildrenCountByDepth[object.__closure[prop].toString()] = 1 } } delete insertionObject.__tree delete insertionObject.__closure insertionObject.totalChildrenCount = closures.length insertionObject.totalChildrenCountByDepth = JSON.stringify( totalChildrenCountByDepth ) await Objects().insert(insertionObject).onConflict().ignore() if (closures.length > 0) { await Closures().insert(closures).onConflict().ignore() } return insertionObject.id } async createObjectsBatched(streamId, objects) { const closures = [] const objsToInsert = [] const ids = [] // Prep objects up objects.forEach((obj) => { const insertionObject = this.prepInsertionObject(streamId, obj) let totalChildrenCountGlobal = 0 const totalChildrenCountByDepth = {} if (obj.__closure !== null) { for (const prop in obj.__closure) { closures.push({ streamId, parent: insertionObject.id, child: prop, minDepth: obj.__closure[prop] }) totalChildrenCountGlobal++ if (totalChildrenCountByDepth[obj.__closure[prop].toString()]) totalChildrenCountByDepth[obj.__closure[prop].toString()]++ else totalChildrenCountByDepth[obj.__closure[prop].toString()] = 1 } } insertionObject.totalChildrenCount = totalChildrenCountGlobal insertionObject.totalChildrenCountByDepth = JSON.stringify( totalChildrenCountByDepth ) delete insertionObject.__tree delete insertionObject.__closure objsToInsert.push(insertionObject) ids.push(insertionObject.id) }) const closureBatchSize = 1000 const objectsBatchSize = 500 // step 1: insert objects if (objsToInsert.length > 0) { const batches = chunk(objsToInsert, objectsBatchSize) for (const [index, batch] of batches.entries()) { this.prepInsertionObjectBatch(batch) await Objects().insert(batch).onConflict().ignore() this.logger.info( { currentBatchCount: batch.length, currentBatchId: index + 1, totalNumberOfBatches: batches.length }, 'Inserted {currentBatchCount} objects from batch {currentBatchId} of {totalNumberOfBatches}' ) } } // step 2: insert closures if (closures.length > 0) { const batches = chunk(closures, closureBatchSize) for (const [index, batch] of batches.entries()) { this.prepInsertionClosureBatch(batch) await Closures().insert(batch).onConflict().ignore() this.logger.info( { currentBatchCount: batch.length, currentBatchId: index + 1, totalNumberOfBatches: batches.length }, 'Inserted {currentBatchCount} closures from batch {currentBatchId} of {totalNumberOfBatches}' ) } } return ids } prepInsertionObject(streamId, obj) { const maximumObjectSizeMB = parseInt(process.env['MAX_OBJECT_SIZE_MB'] || '10') const MAX_OBJECT_SIZE = maximumObjectSizeMB * 1024 * 1024 if (obj.hash) obj.id = obj.hash else obj.id = obj.id || crypto.createHash('md5').update(JSON.stringify(obj)).digest('hex') // generate a hash if none is present const stringifiedObj = JSON.stringify(obj) if (stringifiedObj.length > MAX_OBJECT_SIZE) { throw new Error( `Object too large (${stringifiedObj.length} > ${MAX_OBJECT_SIZE})` ) } return { data: stringifiedObj, // stored in jsonb column streamId, id: obj.id, speckleType: obj.speckleType } } prepInsertionObjectBatch(batch) { batch.sort((a, b) => (a.id > b.id ? 1 : -1)) } prepInsertionClosureBatch(batch) { batch.sort((a, b) => a.parent > b.parent ? 1 : a.parent === b.parent ? a.child > b.child ? 1 : -1 : -1 ) } async getBranchByNameAndStreamId({ streamId, name }) { const query = Branches() .select('*') .where({ streamId }) .andWhere(knex.raw('LOWER(name) = ?', [name])) .first() return await query } async createBranch({ name, description, streamId, authorId }) { const branch = {} branch.id = crs({ length: 10 }) branch.streamId = streamId branch.authorId = authorId branch.name = name.toLowerCase() branch.description = description await Branches().returning('id').insert(branch) // update stream updated at await Streams().where({ id: streamId }).update({ updatedAt: knex.fn.now() }) return branch.id } async createBareToken() { const tokenId = crs({ length: 10 }) const tokenString = crs({ length: 32 }) const tokenHash = await bcrypt.hash(tokenString, 10) const lastChars = tokenString.slice(tokenString.length - 6, tokenString.length) return { tokenId, tokenString, tokenHash, lastChars } } async createToken({ userId, name, scopes, lifespan }) { const { tokenId, tokenString, tokenHash, lastChars } = await this.createBareToken() if (scopes.length === 0) throw new Error('No scopes provided') const token = { id: tokenId, tokenDigest: tokenHash, lastChars, owner: userId, name, lifespan } const tokenScopes = scopes.map((scope) => ({ tokenId, scopeName: scope })) await ApiTokens().insert(token) await TokenScopes().insert(tokenScopes) return { id: tokenId, token: tokenId + tokenString } } async revokeTokenById(tokenId) { const delCount = await ApiTokens() .where({ id: tokenId.slice(0, 10) }) .del() if (delCount === 0) throw new Error('Token revokation failed') return true } }