From 1a4442b5a7876250a6e2b1599ab2db78db90a661 Mon Sep 17 00:00:00 2001 From: Dimitrie Stefanescu Date: Thu, 7 May 2020 21:10:49 +0100 Subject: [PATCH] feat(queries): implemented fast query route for children and started scaffolding the slower, filter enabled one --- modules/core/graph/resolvers/objects.js | 3 + modules/core/graph/schemas/objects.graphql | 24 ++-- modules/core/migrations/000-core.js | 3 - modules/core/objects/services.js | 151 ++++++++++++--------- modules/core/tests/objects.spec.js | 70 ++++++++-- package-lock.json | 5 + package.json | 1 + test-queries/closure-fullcount.sql | 15 +- 8 files changed, 173 insertions(+), 99 deletions(-) diff --git a/modules/core/graph/resolvers/objects.js b/modules/core/graph/resolvers/objects.js index 857543027..272567cbc 100644 --- a/modules/core/graph/resolvers/objects.js +++ b/modules/core/graph/resolvers/objects.js @@ -41,7 +41,10 @@ module.exports = { return await getUser( parent.author ) }, async children( parent, args, context, info ) { + console.log( parent.totalChildrenCount ) console.log( args ) + + throw new ApolloError( 'Not implemented' ) } }, diff --git a/modules/core/graph/schemas/objects.graphql b/modules/core/graph/schemas/objects.graphql index 1a3e7aede..29ac7415c 100644 --- a/modules/core/graph/schemas/objects.graphql +++ b/modules/core/graph/schemas/objects.graphql @@ -47,22 +47,28 @@ type Object { The full object, with all its props & other things. **NOTE:** If you're requesting objects for the purpose of recreating & displaying, you probably only want to request this specific field. """ data: JSON + """ Any objects that this object references. - - `offset`: TODO - - `limit`: TODO - - `depth`: TODO - - `query`: TODO """ - children(offset: Int! = 0, limit: Int! = 100, depth: Int! = 1, query: String, fields: [String] = [], ): [ObjectCollection] + children( + limit: Int! = 100, + depth: Int! = 50, + select: [String], + cursor: String, + query: String, + orderBy: String ): ObjectCollection! + + """ + Query the object's childern, so you can receive only the ones you want to. + """ + childernQuery(limit: Int! = 100, cursor:String, select: [String], depth: Int! = 1 ): ObjectCollection! } type ObjectCollection { totalCount: Int! + cursor: String! + hasMore: Boolean! objects: [Object]! } diff --git a/modules/core/migrations/000-core.js b/modules/core/migrations/000-core.js index 832b54750..3f14a91db 100644 --- a/modules/core/migrations/000-core.js +++ b/modules/core/migrations/000-core.js @@ -75,9 +75,6 @@ exports.up = async knex => { table.index( [ 'speckle_type' ], 'type_index' ) } ) - await knex.raw( 'ALTER TABLE "objects" add column "serial_id" bigserial' ) - await knex.raw( 'CREATE INDEX serial_idx ON objects(serial_id) ' ) - // Tree inheritance tracker await knex.schema.createTable( 'object_tree_refs', table => { table.increments( 'id' ) diff --git a/modules/core/objects/services.js b/modules/core/objects/services.js index a5c1701b4..b5f60fff2 100644 --- a/modules/core/objects/services.js +++ b/modules/core/objects/services.js @@ -3,6 +3,7 @@ const bcrypt = require( 'bcrypt' ) const crs = require( 'crypto-random-string' ) const { performance } = require( 'perf_hooks' ) const crypto = require( 'crypto' ) +const set = require( 'lodash.set' ) let debug = require( 'debug' )( 'speckle:services' ) @@ -67,6 +68,8 @@ module.exports = { }, async createObjects( objects ) { + // TODO: Switch to knex batch inserting functionality + // see http://knexjs.org/#Utility-BatchInsert let batches = [ ] let maxBatchSize = process.env.MAX_BATCH_SIZE || 250 objects = [ ...objects ] @@ -93,13 +96,13 @@ module.exports = { if ( obj.__closure !== null ) { for ( const prop in obj.__closure ) { closures.push( { parent: insertionObject.id, child: prop, minDepth: obj.__closure[ prop ] } ) - + totalChildrenCountGlobal++ - - if( totalChildrenCountByDepth[ obj.__closure[prop].toString() ] ) - totalChildrenCountByDepth[ obj.__closure[ prop ].toString() ]++ - else - totalChildrenCountByDepth[ obj.__closure[ prop ].toString() ] = 1 + + if ( totalChildrenCountByDepth[ obj.__closure[ prop ].toString( ) ] ) + totalChildrenCountByDepth[ obj.__closure[ prop ].toString( ) ]++ + else + totalChildrenCountByDepth[ obj.__closure[ prop ].toString( ) ] = 1 } } @@ -137,78 +140,94 @@ module.exports = { return res }, - async getObjectChildren( objectId, offset, limit, depth, query, fields, orderBy ) { - offset = Math.abs( offset ) || 0 - limit = Math.abs( limit ) || 100 - depth = Math.abs( depth ) || 2 + async getObjectChildren( { objectId, limit, depth, select, cursor } ) { + limit = parseInt( limit ) || 50 + depth = parseInt( depth ) || 1000 - fields = [ 'text', 'nest.flag', 'nest.what', 'arr[1]', 'arr[2]', 'nest.orderMe' ] - let selectFields = `obj_id as id, speckle_type` + let unwrapData = false + let selectStatements = [ ] - fields.forEach( f => { - selectFields += `, jsonb_path_query(data, '$.${ f }') as "data.${f}"` + if ( select && select.length > 0 ) { + selectStatements.push( `jsonb_path_query(data, '$.id') as id` ) + select.forEach( f => { + selectStatements += `, jsonb_path_query(data, '$.${ f }') as "${f}"` + } ) + } else { + selectStatements.push( '"data"' ) + unwrapData = true + } + + let q = Closures( ) + .select( knex.raw( selectStatements ) ) + .rightJoin( 'objects', 'objects.id', 'object_children_closure.child' ) + .where( knex.raw( 'parent = ?', [ objectId ] ) ) + .andWhere( knex.raw( '"minDepth" < ?', [ depth ] ) ) + .andWhere( knex.raw( 'id > ?', [ cursor ? cursor : '0' ] ) ) + .orderBy( 'objects.id' ) + .limit( limit ) + + let rows = await q + + if ( unwrapData ) rows.forEach( ( o, i, arr ) => arr[ i ] = { ...o.data } ) + else rows.forEach( ( o, i, arr ) => { + let no = {} + for ( let key in o ) set( no, key, o[ key ] ) + arr[ i ] = no } ) - orderBy = { property: 'nest.orderMe', direction: 'desc' } + let lastId = rows[ rows.length - 1 ].id + return { rows, cursor: lastId } + }, - // console.log( Refs( ).where( { parent: objectId } ).select( '*' ).toString() ) - // TODO: Analyse and optimise query. - let rawQuery = knex.raw( ` - WITH ids AS ( - SELECT DISTINCT unnest( string_to_array( ltree2text( subltree("path", 1, ${depth}) ), '.') ) as obj_id - FROM object_tree_refs - WHERE parent = '${objectId}' - ), - objs AS ( - SELECT ${selectFields} - FROM ids - JOIN objects ON ids.obj_id = objects.id - -- WHERE objects."data" @> '{"text": "This is object 1"}' - ${ orderBy && orderBy.property && orderBy.direction ? ("ORDER BY jsonb_path_query(data, '$." + orderBy.property + "' ) " + orderBy.direction || "ASC" ) : "ORDER BY obj_id" } - ) - SELECT * from objs - RIGHT JOIN (SELECT count(*) FROM objs) d(totalCount) ON TRUE - OFFSET ${offset} - LIMIT ${limit} - ` ) + async getObjectChildrenQuery( { objectId, limit, depth, select, cursor, query } ) { + limit = parseInt( limit ) || 50 + depth = parseInt( depth ) || 1000 - let betterQuery = ` - WITH ids AS( - SELECT unnest( string_to_array( ltree2text( subltree("path", 1, 2) ), '.') ) as obj_id - FROM object_tree_refs - -- WHERE path ~ '0_hash.*{1}' - WHERE nlevel(path) = 2 - ), - objs AS( - SELECT obj_id, speckle_type, serial_id, - jsonb_path_query(data, '$.text') as "data.text", - jsonb_path_query(data, '$.nest.flag') as "data.nest.flag", - jsonb_path_query(data, '$.nest.what') as "data.nest.what", - jsonb_path_query(data, '$.arr[1]') as "data.arr[1]", - jsonb_path_query(data, '$.arr[2]') as "data.arr[2]", - jsonb_path_query(data, '$.nest.orderMe') as "data.nest.orderMe" - FROM ids - JOIN objects ON ids.obj_id = objects.id - -- WHERE (objects."data" -> 'nest' ->> 'orderMe')::numeric >= 19001 - -- AND (objects."data"->'nest'->>'what') LIKE '%42%' - ) - SELECT * FROM objs - RIGHT JOIN (SELECT count(*) FROM objs ) c(total_count) ON TRUE - ORDER BY serial_id desc - OFFSET 310 - LIMIT 1000 - ` + let unwrapData = false + let q = knex.with( 'objs', qb => { + qb.select( 'id' ).from( 'object_children_closure' ) + if ( select && select.length > 0 ) { + select.forEach( ( field, index ) => { + qb.select( knex.raw( 'jsonb_path_query(data, :path) as :name:', { path: "$." + field, name: '' + index } ) ) + } ) + } else { + unwrapData = true + qb.select( 'data' ) + } + qb.join( 'objects', 'child', '=', 'objects.id' ) + .where( 'parent', objectId ) - console.log( rawQuery.toString( ) ) + if( query && query.length > 0) { + query.forEach( statement => { + // qb.andWhere() + }) + } - let t0 = performance.now( ) + } ) + .select( '*' ).from( 'objs' ) + .joinRaw( 'RIGHT JOIN ( SELECT count(*) FROM "objs" ) c(total_count) ON TRUE' ) + // .orderBy() // TODO + .limit( 2 ) - let res = await rawQuery + console.log( q.toString( ) ) - let t1 = performance.now( ) + let rows = await q + console.log( rows ) + let totalCount = rows && rows.length > 0 ? rows[ 0 ].total_count : 0 - console.log( `Found ${res.rows.length} in ${t1-t0}ms.` ) + if ( unwrapData ) rows.forEach( ( o, i, arr ) => arr[ i ] = { ...o.data } ) + else { + rows.forEach( ( o, i, arr ) => { + let no = {} + let k = 0 + for ( let field of select ) { + set( no, field, o[ k++ ] ) + } + arr[ i ] = no + } ) + } + console.log( rows ) }, async getObjects( objectIds ) { diff --git a/modules/core/tests/objects.spec.js b/modules/core/tests/objects.spec.js index 8fa4f3f91..ba37e0bce 100644 --- a/modules/core/tests/objects.spec.js +++ b/modules/core/tests/objects.spec.js @@ -12,7 +12,7 @@ chai.use( chaiHttp ) const { createUser, createToken, revokeToken, revokeTokenById, validateToken, getUserTokens } = require( '../users/services' ) const { createStream, getStream, updateStream, deleteStream, getStreamsUser, grantPermissionsStream, revokePermissionsStream } = require( '../streams/services' ) -const { createCommit, createObject, createObjects, getObject, getObjects, getObjectChildren } = require( '../objects/services' ) +const { createCommit, createObject, createObjects, getObject, getObjects, getObjectChildren, getObjectChildrenQuery } = require( '../objects/services' ) const sampleObjects = require( './sampleObjectData' ) @@ -154,23 +154,57 @@ describe( 'Objects', ( ) => { expect( match2.id ).to.equal( objs[ 2 ].id ) } ) + let parentObjectId + it( 'Should get object children', async ( ) => { - let objs_1 = createAShitTonOfFuckingObjects( 10000, 'noise__' ) + let objs_1 = createManyObjects( 100, 'noise__' ) let ids = await createObjects( objs_1 ) - // let objs_2 = createAShitTonOfFuckingObjects( 20000, 'noise_2' ) + // let objs_2 = createManyObjects( 20000, 'noise_2' ) // let ids2 = await createObjects( objs_2 ) - // let objs_3 = createAShitTonOfFuckingObjects( 50000, 'noise_3' ) + // let objs_3 = createManyObjects( 100000, 'noise_3' ) // let ids3 = await createObjects( objs_3 ) - - console.log( `base id is: ${ids[0]} ` ) - console.log( `base id is: ${ids2[0]} ` ) - console.log( `base id is: ${ids3[0]} ` ) + + + + // let { rows } = await getObjectChildren( { objectId: ids[0], select: ['id', 'name', 'sortValueB'] } ) + // let { rows } = await getObjectChildren( { objectId: ids[ 0 ] } ) + let limit = 50 + let { rows: rows_1, cursor: cursor_1 } = await getObjectChildren( { limit, objectId: ids[ 0 ], select: [ 'nest.mallard', 'test.value', 'test.secondValue', 'nest.arr[0]', 'nest.arr[1]' ] } ) + + expect( rows_1.length ).to.equal( limit ) + expect( rows_1[ 0 ] ).to.be.an( 'object' ) + expect( rows_1[ 0 ] ).to.have.property( 'id' ) + expect( rows_1[ 0 ] ).to.have.nested.property( 'test.secondValue' ) + expect( rows_1[ 0 ] ).to.have.nested.property( 'nest.mallard' ) + + expect( cursor_1 ).to.be.a( 'string' ) + + let { rows: rows_2, cursor: cursor_2 } = await getObjectChildren( { limit, objectId: ids[ 0 ], select: [ 'nest.mallard', 'test.value', 'test.secondValue', 'nest.arr[0]', 'nest.arr[1]' ], cursor: cursor_1 } ) + + expect( rows_2.length ).to.equal( 50 ) + expect( rows_2[ 0 ] ).to.be.an( 'object' ) + expect( rows_2[ 0 ] ).to.have.property( 'id' ) + expect( rows_2[ 0 ] ).to.have.nested.property( 'test.secondValue' ) + expect( rows_2[ 0 ] ).to.have.nested.property( 'nest.mallard' ) + + + let { rows, cursor } = await getObjectChildren( { objectId: ids[ 0 ], limit: 1000 } ) + expect( rows.length ).to.equal( 100 ) + + parentObjectId = ids[ 0 ] } ).timeout( 30000 ) + it( 'should query object children', async ( ) => { + // we're assuming the prev test objects exist + + let test = await getObjectChildrenQuery( { objectId: parentObjectId, select: [ 'nest.mallard', 'test.value' ] } ) + + } ) + } ) describe( 'Integration (API)', ( ) => { @@ -256,9 +290,9 @@ describe( 'Objects', ( ) => { const crypto = require( 'crypto' ) -function createAShitTonOfFuckingObjects( shitTon, noise ) { +function createManyObjects( shitTon, noise ) { shitTon = shitTon || 10000 - noise = noise || Math.random() * 100 + noise = noise || Math.random( ) * 100 let objs = [ ] @@ -266,13 +300,21 @@ function createAShitTonOfFuckingObjects( shitTon, noise ) { objs.push( base ) for ( let i = 0; i < shitTon; i++ ) { - let baby = { name: `mr. ${i}`, noise: noise, sortValueA: i, sortValueB: i * 0.42 * i } + let baby = { + name: `mr. ${i}`, + nest: { duck: true, mallard: 'false', arr: [ i + 42, i, i ] }, + test: { value: i, secondValue: 'mallard ' + i % 10 }, + objArr: [ { a: i }, { b: i * i }, { c: true } ], + noise: noise, + sortValueA: i, + sortValueB: i * 0.42 * i + } getAFuckingId( baby ) base.__closure[ baby.id ] = 1 - - if( i > 1000 ) + + if ( i > 1000 ) base.__closure[ baby.id ] = i / 1000 - + objs.push( baby ) } diff --git a/package-lock.json b/package-lock.json index 097cf23d1..c80969a57 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3392,6 +3392,11 @@ "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==" }, + "lodash.set": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz", + "integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=" + }, "lodash.sortby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", diff --git a/package.json b/package.json index 1e109520e..e6bfc9f72 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "graphql-tools": "^4.0.7", "knex": "^0.20.12", "lodash.merge": "^4.6.2", + "lodash.set": "^4.3.2", "lodash.values": "^4.3.0", "morgan": "^1.10.0", "morgan-debug": "^2.0.0", diff --git a/test-queries/closure-fullcount.sql b/test-queries/closure-fullcount.sql index e42183379..1c414bc0f 100644 --- a/test-queries/closure-fullcount.sql +++ b/test-queries/closure-fullcount.sql @@ -1,18 +1,19 @@ with objs as ( SELECT --- child as id, id, - serial_id, -- just for reference "data" FROM object_children_closure JOIN objects ON objects.id = child - WHERE parent = '7919a52c017be262ee0daf1844c376d7' - AND "minDepth" < 1000 --- AND (objects."data" -> 'sortValueA')::numeric <= 700 + WHERE parent = '89b42e4109f32b20763243d4313e81b5' +-- AND "minDepth" < 1000 + AND (jsonb_path_query("data", '$.sortValueA.test.value'))::numeric <= 10 -- AND (objects."data" -> 'sortValueA')::numeric > 100 ORDER BY id ) SELECT * FROM objs RIGHT JOIN (SELECT count(*) FROM objs ) c(total_count) ON TRUE -OFFSET 100 -LIMIT 200 \ No newline at end of file +OFFSET 0 +LIMIT 200 + + +-- with "objs" as (select *) select * from "objs" RIGHT JOIN ( SELECT count(*) FROM "objs" ) c(total_count) ON TRUE limit 50