Merge pull request #287 from specklesystems/dim/stats-endpoints

Dim/stats endpoints

packages/server/app.js

@@ -117,9 +117,12 @@ exports.init = async ( ) => {
  * @param  {[type]} app [description]
  * @return {[type]}     [description]
  */
-exports.startHttp = async ( app ) => {
+exports.startHttp = async ( app, customPortOverride ) => {
   let bindAddress = process.env.BIND_ADDRESS || '127.0.0.1'
   let port = process.env.PORT || 3000
+
+  if ( customPortOverride ) port = customPortOverride
+
   app.set( 'port', port )
 
   let frontendHost = process.env.FRONTEND_HOST || 'localhost'

packages/server/modules/core/scopes.js

@@ -31,6 +31,11 @@ module.exports = [
     description: 'Read other users\' profile on your behalf.',
     public: true
   },
+  {
+    name: 'server:stats',
+    description: 'Request server stats from the api. Only works in conjunction with a "server:admin" role.',
+    public: true
+  },
   {
     name: 'users:email',
     description: 'Access the emails of other users on your behalf.',

packages/server/modules/core/tests/objects.spec.js

@@ -480,8 +480,8 @@ describe( 'Objects @core-objects', ( ) => {
 
 const crypto = require( 'crypto' )
 
-function createManyObjects( shitTon, noise ) {
-  shitTon = shitTon || 10000
+function createManyObjects( num, noise ) {
+  num = num || 10000
   noise = noise || Math.random( ) * 100
 
   let objs = [ ]
@@ -490,7 +490,7 @@ function createManyObjects( shitTon, noise ) {
   objs.push( base )
   let k = 0
 
-  for ( let i = 0; i < shitTon; i++ ) {
+  for ( let i = 0; i < num; i++ ) {
     let baby = {
       name: `mr. ${i}`,
       nest: { duck: i % 2 === 0, mallard: 'falsey', arr: [ i + 42, i, i ] },
@@ -504,7 +504,7 @@ function createManyObjects( shitTon, noise ) {
     }
 
     if ( i % 3 === 0 ) k++
-    getAFuckingId( baby )
+    getAnId( baby )
 
     base.__closure[ baby.id ] = 1
     if ( i > 1000 )
@@ -513,10 +513,10 @@ function createManyObjects( shitTon, noise ) {
     objs.push( baby )
   }
 
-  getAFuckingId( base )
+  getAnId( base )
   return objs
 }
 
-function getAFuckingId( obj ) {
+function getAnId( obj ) {
   obj.id = obj.id || crypto.createHash( 'md5' ).update( JSON.stringify( obj ) ).digest( 'hex' )
 }

packages/server/modules/stats/graph/resolvers/stats.js

@@ -0,0 +1,50 @@
+'use strict'
+const appRoot = require( 'app-root-path' )
+const { validateServerRole, validateScopes } = require( `${appRoot}/modules/shared` )
+const { getStreamHistory, getCommitHistory, getObjectHistory, getUserHistory, getTotalStreamCount, getTotalCommitCount, getTotalObjectCount, getTotalUserCount } = require( '../../services' )
+
+
+module.exports = {
+  Query: {
+    async serverStats( parent, args, context, info ) {
+
+      await validateServerRole( context, 'server:admin' )
+      await validateScopes( context.scopes, 'server:stats' )
+      return {}
+    }
+  },
+
+  ServerStats: {
+    async totalStreamCount() {
+      return await getTotalStreamCount()
+    },
+
+    async totalCommitCount() {
+      return await getTotalCommitCount()
+    },
+
+    async totalObjectCount() {
+      return await getTotalObjectCount()
+    },
+
+    async totalUserCount() {
+      return await getTotalUserCount()
+    },
+
+    async streamHistory() {
+      return await getStreamHistory()
+    },
+
+    async commitHistory() {
+      return await getCommitHistory()
+    },
+
+    async objectHistory() {
+      return await getObjectHistory()
+    },
+
+    async userHistory() {
+      return await getUserHistory()
+    }
+  }
+}

packages/server/modules/stats/graph/schemas/stats.gql

@@ -0,0 +1,26 @@
+extend type Query {
+  serverStats: ServerStats!
+}
+
+type ServerStats {
+  totalStreamCount: Int!
+  totalCommitCount: Int!
+  totalObjectCount: Int!
+  totalUserCount: Int!
+  """
+  An array of objects currently structured as { created_month: Date, count: int }.
+  """
+  streamHistory: [JSONObject]
+  """
+  An array of objects currently structured as { created_month: Date, count: int }.
+  """
+  commitHistory: [JSONObject]
+  """
+  An array of objects currently structured as { created_month: Date, count: int }.
+  """
+  objectHistory: [JSONObject]
+  """
+  An array of objects currently structured as { created_month: Date, count: int }.
+  """
+  userHistory: [JSONObject]
+}

packages/server/modules/stats/index.js

@@ -0,0 +1,13 @@
+'use strict'
+let debug = require( 'debug' )
+const appRoot = require( 'app-root-path' )
+const { registerOrUpdateScope } = require( `${appRoot}/modules/shared` )
+
+exports.init = async ( app, options ) => {
+  debug( 'speckle:modules' )( '📊 Init stats module' )
+  // TODO
+}
+
+exports.finalize = async () => {
+  // TODO
+}

packages/server/modules/stats/services/index.js

@@ -0,0 +1,93 @@
+'use strict'
+const appRoot = require( 'app-root-path' )
+const knex = require( `${appRoot}/db/knex` )
+
+module.exports = {
+
+  async getTotalStreamCount() {
+    const query = 'SELECT COUNT(*) FROM streams'
+    const result = await knex.raw( query )
+    return parseInt( result.rows[0].count )
+  },
+
+  async getTotalCommitCount() {
+    const query = 'SELECT COUNT(*) FROM commits'
+    const result = await knex.raw( query )
+    return parseInt( result.rows[0].count )
+  },
+
+  async getTotalObjectCount() {
+    const query = 'SELECT COUNT(*) FROM objects'
+    const result = await knex.raw( query )
+    return parseInt( result.rows[0].count )
+  },
+
+  async getTotalUserCount() {
+    const query = 'SELECT COUNT(*) FROM users'
+    const result = await knex.raw( query )
+    return parseInt( result.rows[0].count )
+  },
+
+  async getStreamHistory() {
+    const query = `
+    SELECT
+      DATE_TRUNC('month', streams. "createdAt") AS created_month,
+      COUNT(id) AS count
+    FROM
+      streams
+    GROUP BY
+      DATE_TRUNC('month', streams. "createdAt")
+    `
+
+    const result = await knex.raw( query )
+    result.rows.forEach( row => row.count = parseInt( row.count ) )
+    return result.rows
+  },
+
+  async getCommitHistory() {
+    const query = `
+    SELECT
+      DATE_TRUNC('month', commits. "createdAt") AS created_month,
+      COUNT(id) AS count
+    FROM
+      commits
+    GROUP BY
+      DATE_TRUNC('month', commits. "createdAt")
+    `
+    const result = await knex.raw( query )
+    result.rows.forEach( row => row.count = parseInt( row.count ) )
+    return result.rows
+  },
+
+  async getObjectHistory() {
+    const query = `
+    SELECT
+      DATE_TRUNC('month', objects. "createdAt") AS created_month,
+      COUNT(id) AS count
+    FROM
+      objects
+    GROUP BY
+      DATE_TRUNC('month', objects. "createdAt")
+    `
+    const result = await knex.raw( query )
+    result.rows.forEach( row => row.count = parseInt( row.count ) )
+    return result.rows
+
+  },
+
+  async getUserHistory() {
+    const query = `
+    SELECT
+      DATE_TRUNC('month', users. "createdAt") AS created_month,
+      COUNT(id) AS count
+    FROM
+      users
+    GROUP BY
+      DATE_TRUNC('month', users. "createdAt")
+    `
+    const result = await knex.raw( query )
+    result.rows.forEach( row => row.count = parseInt( row.count ) )
+    return result.rows
+  },
+
+}

packages/server/modules/stats/tests/stats.spec.js

@@ -0,0 +1,263 @@
+/* istanbul ignore file */
+const chai = require( 'chai' )
+const chaiHttp = require( 'chai-http' )
+const assert = require( 'assert' )
+
+const appRoot = require( 'app-root-path' )
+const { init, startHttp } = require( `${appRoot}/app` )
+const knex = require( `${appRoot}/db/knex` )
+
+const expect = chai.expect
+chai.use( chaiHttp )
+
+const crypto = require( 'crypto' )
+const { createUser } = require( `${appRoot}/modules/core/services/users` )
+const { createPersonalAccessToken } = require( `${appRoot}/modules/core/services/tokens` )
+const { createStream } = require( `${appRoot}/modules/core/services/streams` )
+const { createObjects } = require( `${appRoot}/modules/core/services/objects` )
+const { createCommitByBranchName, createCommitByBranchId } = require( `${appRoot}/modules/core/services/commits` )
+
+const { getStreamHistory, getCommitHistory, getObjectHistory, getUserHistory, getTotalStreamCount, getTotalCommitCount, getTotalObjectCount, getTotalUserCount } = require( '../services' )
+
+const params = { numUsers: 25, numStreams: 30, numObjects: 100, numCommits: 100 }
+
+describe( 'Server stats services @stats-services', function() {
+
+  before( async function() {
+    this.timeout( 10000 )
+
+    await knex.migrate.rollback( )
+    await knex.migrate.latest( )
+
+    await init()
+    await seedDb( params )
+  } )
+
+  after( async() => {
+    await knex.migrate.rollback( )
+  } )
+
+  it( 'should return the total number of users on this server', async () => {
+    let res = await getTotalUserCount()
+    expect( res ).to.equal( params.numUsers )
+  } )
+
+  it( 'should return the total number of streams on this server', async() => {
+    let res = await getTotalStreamCount()
+    expect( res ).to.equal( params.numStreams )
+  } )
+
+  it( 'should return the total number of commits on this server', async() => {
+    let res = await getTotalCommitCount()
+    expect( res ).to.equal( params.numCommits )
+  } )
+
+  it( 'should return the total number of objects on this server', async() => {
+    let res = await getTotalObjectCount()
+    expect( res ).to.equal( params.numObjects )
+  } )
+
+  it( 'should return the stream creation history by month', async() => {
+    let res = await getStreamHistory()
+    expect( res ).to.be.an( 'array' )
+    expect( res[0] ).to.have.property( 'count' )
+    expect( res[0] ).to.have.property( 'created_month' )
+    expect( res[0].count ).to.be.a( 'number' )
+    expect( res[0].count ).to.equal( params.numStreams )
+  } )
+
+  it( 'should return the commit creation history by month', async() => {
+    let res = await getCommitHistory()
+    expect( res ).to.be.an( 'array' )
+    expect( res[0] ).to.have.property( 'count' )
+    expect( res[0] ).to.have.property( 'created_month' )
+    expect( res[0].count ).to.be.a( 'number' )
+    expect( res[0].count ).to.equal( params.numCommits )
+  } )
+
+  it( 'should return the object creation history by month', async() => {
+    let res = await getObjectHistory()
+    expect( res ).to.be.an( 'array' )
+    expect( res[0] ).to.have.property( 'count' )
+    expect( res[0] ).to.have.property( 'created_month' )
+    expect( res[0].count ).to.be.a( 'number' )
+    expect( res[0].count ).to.equal( params.numObjects )
+  } )
+
+  it( 'should return the user creation history by month', async() => {
+    let res = await getUserHistory()
+    expect( res ).to.be.an( 'array' )
+    expect( res[0] ).to.have.property( 'count' )
+    expect( res[0] ).to.have.property( 'created_month' )
+    expect( res[0].count ).to.be.a( 'number' )
+    expect( res[0].count ).to.equal( params.numUsers )
+  } )
+
+} )
+
+let addr = `http://localhost:3333`
+
+describe( 'Server stats api @stats-api', function() {
+  let testServer
+  let adminUser = {
+    name: 'Dimitrie',
+    password: 'TestPasswordSecure',
+    email: 'spam@spam.spam'
+  }
+
+  let notAdminUser = {
+    name: 'Andrei',
+    password: 'TestPasswordSecure',
+    email: 'spasm@spam.spam'
+  }
+
+  let fullQuery = `
+  query{
+    serverStats{
+      totalStreamCount
+      totalCommitCount
+      totalObjectCount
+      totalUserCount
+      streamHistory
+      commitHistory
+      objectHistory
+      userHistory
+      }
+    }
+    `
+
+  before( async function() {
+    this.timeout( 10000 )
+    await knex.migrate.rollback( )
+    await knex.migrate.latest( )
+
+    let { app } = await init( )
+    let { server } = await startHttp( app, 3333 )
+    testServer = server
+
+    adminUser.id = await createUser( adminUser )
+    adminUser.goodToken = `Bearer ${( await createPersonalAccessToken( adminUser.id, 'test token user A', [ 'server:stats' ] ) )}`
+    adminUser.badToken = `Bearer ${( await createPersonalAccessToken( adminUser.id, 'test token user A', [ 'streams:read' ] ) )}`
+
+    notAdminUser.id = await createUser( notAdminUser )
+    notAdminUser.goodToken = `Bearer ${( await createPersonalAccessToken( notAdminUser.id, 'test token user A', [ 'server:stats' ] ) )}`
+    notAdminUser.badToken = `Bearer ${( await createPersonalAccessToken( notAdminUser.id, 'test token user A', [ 'streams:read' ] ) )}`
+
+    await seedDb( params )
+
+  } )
+
+  after( async function() {
+    await knex.migrate.rollback( )
+    testServer.close( )
+  } )
+
+  it( 'Should not get stats if user is not admin', async() => {
+
+    let res = await sendRequest( adminUser.badToken, { query: fullQuery } )
+    expect( res.body.errors ).to.exist
+    expect( res.body.errors[0].extensions.code ).to.equal( 'FORBIDDEN' )
+
+  } )
+
+  it( 'Should not get stats if user is not admin even if the token has the correct scopes', async() => {
+    let res = await sendRequest( notAdminUser.goodToken, { query: fullQuery } )
+    expect( res.body.errors ).to.exist
+    expect( res.body.errors[0].extensions.code ).to.equal( 'FORBIDDEN' )
+  } )
+
+  it( 'Should not get stats if token does not have required scope', async() => {
+    let res = await sendRequest( adminUser.badToken, { query: fullQuery } )
+    expect( res ).to.be.json
+    expect( res.body.errors ).to.exist
+    expect( res.body.errors[0].extensions.code ).to.equal( 'FORBIDDEN' )
+  } )
+
+  it( 'Should get server stats', async() => {
+    let res = await sendRequest( adminUser.goodToken, { query: fullQuery } )
+    expect( res ).to.be.json
+    expect( res.body.errors ).to.not.exist
+
+    expect( res.body.data ).to.have.property( 'serverStats' )
+    expect( res.body.data.serverStats ).to.have.property( 'totalStreamCount' )
+    expect( res.body.data.serverStats ).to.have.property( 'totalCommitCount' )
+    expect( res.body.data.serverStats ).to.have.property( 'totalObjectCount' )
+    expect( res.body.data.serverStats ).to.have.property( 'totalUserCount' )
+    expect( res.body.data.serverStats ).to.have.property( 'streamHistory' )
+    expect( res.body.data.serverStats ).to.have.property( 'commitHistory' )
+    expect( res.body.data.serverStats ).to.have.property( 'userHistory' )
+
+    expect( res.body.data.serverStats.totalStreamCount ).to.equal( params.numStreams )
+    expect( res.body.data.serverStats.totalCommitCount ).to.equal( params.numCommits )
+    expect( res.body.data.serverStats.totalObjectCount ).to.equal( params.numObjects )
+    expect( res.body.data.serverStats.totalUserCount ).to.equal( params.numUsers + 2 ) // we're registering two extra users in the before hook
+
+    expect( res.body.data.serverStats.streamHistory ).to.be.an( 'array' )
+    expect( res.body.data.serverStats.commitHistory ).to.be.an( 'array' )
+    expect( res.body.data.serverStats.userHistory ).to.be.an( 'array' )
+
+  } )
+
+} )
+
+async function seedDb( { numUsers = 10, numStreams = 10, numObjects = 10, numCommits = 10 } = {} ) {
+  let users = []
+  let streams = []
+
+  // create users
+  for ( let i = 0; i < numUsers; i++ ) {
+    let id = await createUser( { name: `User ${i}`, password: `SuperSecure${i}${i*3.14}`, email: `user${i}@speckle.systems` } )
+    users.push( id )
+  }
+
+  // create streams
+  for ( let i = 0; i < numStreams; i++ ) {
+    let id = await createStream( { name: `Stream ${i}`, ownerId: users[i >= users.length ? users.length - 1 : i ] } )
+    streams.push( id )
+  }
+
+  // create a objects
+  let mockObjects = createManyObjects( numObjects - 1 )
+  let objs = await createObjects( streams[ 0 ], mockObjects )
+  let commits = []
+
+  // create commits referencing those objects
+  for ( let i = 0; i < numCommits; i++ ) {
+    let id = await createCommitByBranchName( {
+      streamId: streams[0],
+      branchName: 'main',
+      sourceApplication: 'tests',
+      objectId: objs[ i >= objs.length ? objs.length - 1 : i ]
+    } )
+    commits.push( id )
+  }
+
+}
+
+function createManyObjects( num, noise ) {
+  num = num || 10000
+  noise = noise || Math.random( ) * 100
+
+  let objs = [ ]
+
+  let base = { name: 'base bastard 2', noise: noise, __closure: {} }
+  objs.push( base )
+  let k = 0
+
+  for ( let i = 0; i < num; i++ ) {
+    let baby = { name: `mr. ${i}`, nest: { duck: i % 2 === 0, mallard: 'falsey', arr: [ i + 42, i, i ] } }
+    getAnId( baby )
+    base.__closure[ baby.id ] = 1
+    objs.push( baby )
+  }
+  getAnId( base )
+  return objs
+}
+
+function getAnId( obj ) {
+  obj.id = obj.id || crypto.createHash( 'md5' ).update( JSON.stringify( obj ) ).digest( 'hex' )
+}
+
+function sendRequest( auth, obj, address = addr ) {
+  return chai.request( address ).post( '/graphql' ).set( 'Authorization', auth ).send( obj )
+}