huanld/speckle-server
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(queries): implemented fast query route for children

Browse Source 
and started scaffolding the slower, filter enabled one
This commit is contained in:
Dimitrie Stefanescu
2020-05-07 21:10:49 +01:00
parent 85cc525663
commit 1a4442b5a7
8 changed files with 173 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/core/graph/resolvers/objects.js
+3
View File
@@ -41,7 +41,10 @@ module.exports = {
return await getUser( parent.author )
},
async children( parent, args, context, info ) {
console.log( parent.totalChildrenCount )
console.log( args )
throw new ApolloError( 'Not implemented' )
}
},
modules/core/graph/schemas/objects.graphql
+15 -9
View File
@@ -47,22 +47,28 @@ type Object {
The full object, with all its props & other things. **NOTE:** If you're requesting objects for the purpose of recreating & displaying, you probably only want to request this specific field.
"""
data: JSON
"""
Any objects that this object references.
`offset`: TODO
`limit`: TODO
`depth`: TODO
`query`: TODO
"""
children(offset: Int! = 0, limit: Int! = 100, depth: Int! = 1, query: String, fields: [String] = [], ): [ObjectCollection]
children(
limit: Int! = 100,
depth: Int! = 50,
select: [String],
cursor: String,
query: String,
orderBy: String ): ObjectCollection!
"""
Query the object's childern, so you can receive only the ones you want to.
"""
childernQuery(limit: Int! = 100, cursor:String, select: [String], depth: Int! = 1 ): ObjectCollection!
}
type ObjectCollection {
totalCount: Int!
cursor: String!
hasMore: Boolean!
objects: [Object]!
}
modules/core/migrations/000-core.js
-3
View File
@@ -75,9 +75,6 @@ exports.up = async knex => {
table.index( [ 'speckle_type' ], 'type_index' )
} )
await knex.raw( 'ALTER TABLE "objects" add column "serial_id" bigserial' )
await knex.raw( 'CREATE INDEX serial_idx ON objects(serial_id) ' )
// Tree inheritance tracker
await knex.schema.createTable( 'object_tree_refs', table => {
table.increments( 'id' )
modules/core/objects/services.js
+85 -66
View File
@@ -3,6 +3,7 @@ const bcrypt = require( 'bcrypt' )
const crs = require( 'crypto-random-string' )
const { performance } = require( 'perf_hooks' )
const crypto = require( 'crypto' )
const set = require( 'lodash.set' )
let debug = require( 'debug' )( 'speckle:services' )
@@ -67,6 +68,8 @@ module.exports = {
},
async createObjects( objects ) {
// TODO: Switch to knex batch inserting functionality
// see http://knexjs.org/#Utility-BatchInsert
let batches = [ ]
let maxBatchSize = process.env.MAX_BATCH_SIZE || 250
objects = [ ...objects ]
@@ -93,13 +96,13 @@ module.exports = {
if ( obj.__closure !== null ) {
for ( const prop in obj.__closure ) {
closures.push( { parent: insertionObject.id, child: prop, minDepth: obj.__closure[ prop ] } )
totalChildrenCountGlobal++
if( totalChildrenCountByDepth[ obj.__closure[prop].toString() ] )
totalChildrenCountByDepth[ obj.__closure[ prop ].toString() ]++
else
totalChildrenCountByDepth[ obj.__closure[ prop ].toString() ] = 1
if ( totalChildrenCountByDepth[ obj.__closure[ prop ].toString( ) ] )
totalChildrenCountByDepth[ obj.__closure[ prop ].toString( ) ]++
else
totalChildrenCountByDepth[ obj.__closure[ prop ].toString( ) ] = 1
}
}
@@ -137,78 +140,94 @@ module.exports = {
return res
},
async getObjectChildren( objectId, offset, limit, depth, query, fields, orderBy ) {
offset = Math.abs( offset ) || 0
limit = Math.abs( limit ) || 100
depth = Math.abs( depth ) || 2
async getObjectChildren( { objectId, limit, depth, select, cursor } ) {
limit = parseInt( limit ) || 50
depth = parseInt( depth ) || 1000
fields = [ 'text', 'nest.flag', 'nest.what', 'arr[1]', 'arr[2]', 'nest.orderMe' ]
let selectFields = `obj_id as id, speckle_type`
let unwrapData = false
let selectStatements = [ ]
fields.forEach( f => {
selectFields += `, jsonb_path_query(data, '$.${ f }') as "data.${f}"`
if ( select && select.length > 0 ) {
selectStatements.push( `jsonb_path_query(data, '$.id') as id` )
select.forEach( f => {
selectStatements += `, jsonb_path_query(data, '$.${ f }') as "${f}"`
} )
} else {
selectStatements.push( '"data"' )
unwrapData = true
}
let q = Closures( )
.select( knex.raw( selectStatements ) )
.rightJoin( 'objects', 'objects.id', 'object_children_closure.child' )
.where( knex.raw( 'parent = ?', [ objectId ] ) )
.andWhere( knex.raw( '"minDepth" < ?', [ depth ] ) )
.andWhere( knex.raw( 'id > ?', [ cursor ? cursor : '0' ] ) )
.orderBy( 'objects.id' )
.limit( limit )
let rows = await q
if ( unwrapData ) rows.forEach( ( o, i, arr ) => arr[ i ] = { ...o.data } )
else rows.forEach( ( o, i, arr ) => {
let no = {}
for ( let key in o ) set( no, key, o[ key ] )
arr[ i ] = no
} )
orderBy = { property: 'nest.orderMe', direction: 'desc' }
let lastId = rows[ rows.length - 1 ].id
return { rows, cursor: lastId }
},
// console.log( Refs( ).where( { parent: objectId } ).select( '*' ).toString() )
// TODO: Analyse and optimise query.
let rawQuery = knex.raw( `
WITH ids AS (
SELECT DISTINCT unnest( string_to_array( ltree2text( subltree("path", 1, ${depth}) ), '.') ) as obj_id
FROM object_tree_refs
WHERE parent = '${objectId}'
),
objs AS (
SELECT ${selectFields}
FROM ids
JOIN objects ON ids.obj_id = objects.id
-- WHERE objects."data" @> '{"text": "This is object 1"}'
${ orderBy && orderBy.property && orderBy.direction ? ("ORDER BY jsonb_path_query(data, '$." + orderBy.property + "' ) " + orderBy.direction || "ASC" ) : "ORDER BY obj_id" }
)
SELECT * from objs
RIGHT JOIN (SELECT count(*) FROM objs) d(totalCount) ON TRUE
OFFSET ${offset}
LIMIT ${limit}
` )
async getObjectChildrenQuery( { objectId, limit, depth, select, cursor, query } ) {
limit = parseInt( limit ) || 50
depth = parseInt( depth ) || 1000
let betterQuery = `
WITH ids AS(
SELECT unnest( string_to_array( ltree2text( subltree("path", 1, 2) ), '.') ) as obj_id
FROM object_tree_refs
-- WHERE path ~ '0_hash.*{1}'
WHERE nlevel(path) = 2
),
objs AS(
SELECT obj_id, speckle_type, serial_id,
jsonb_path_query(data, '$.text') as "data.text",
jsonb_path_query(data, '$.nest.flag') as "data.nest.flag",
jsonb_path_query(data, '$.nest.what') as "data.nest.what",
jsonb_path_query(data, '$.arr[1]') as "data.arr[1]",
jsonb_path_query(data, '$.arr[2]') as "data.arr[2]",
jsonb_path_query(data, '$.nest.orderMe') as "data.nest.orderMe"
FROM ids
JOIN objects ON ids.obj_id = objects.id
-- WHERE (objects."data" -> 'nest' ->> 'orderMe')::numeric >= 19001
-- AND (objects."data"->'nest'->>'what') LIKE '%42%'
)
SELECT * FROM objs
RIGHT JOIN (SELECT count(*) FROM objs ) c(total_count) ON TRUE
ORDER BY serial_id desc
OFFSET 310
LIMIT 1000
`
let unwrapData = false
let q = knex.with( 'objs', qb => {
qb.select( 'id' ).from( 'object_children_closure' )
if ( select && select.length > 0 ) {
select.forEach( ( field, index ) => {
qb.select( knex.raw( 'jsonb_path_query(data, :path) as :name:', { path: "$." + field, name: '' + index } ) )
} )
} else {
unwrapData = true
qb.select( 'data' )
}
qb.join( 'objects', 'child', '=', 'objects.id' )
.where( 'parent', objectId )
console.log( rawQuery.toString( ) )
if( query && query.length > 0) {
query.forEach( statement => {
// qb.andWhere()
})
}
let t0 = performance.now( )
} )
.select( '*' ).from( 'objs' )
.joinRaw( 'RIGHT JOIN ( SELECT count(*) FROM "objs" ) c(total_count) ON TRUE' )
// .orderBy() // TODO
.limit( 2 )
let res = await rawQuery
console.log( q.toString( ) )
let t1 = performance.now( )
let rows = await q
console.log( rows )
let totalCount = rows && rows.length > 0 ? rows[ 0 ].total_count : 0
console.log( `Found ${res.rows.length} in ${t1-t0}ms.` )
if ( unwrapData ) rows.forEach( ( o, i, arr ) => arr[ i ] = { ...o.data } )
else {
rows.forEach( ( o, i, arr ) => {
let no = {}
let k = 0
for ( let field of select ) {
set( no, field, o[ k++ ] )
}
arr[ i ] = no
} )
}
console.log( rows )
},
async getObjects( objectIds ) {
modules/core/tests/objects.spec.js
+56 -14
View File
@@ -12,7 +12,7 @@ chai.use( chaiHttp )
const { createUser, createToken, revokeToken, revokeTokenById, validateToken, getUserTokens } = require( '../users/services' )
const { createStream, getStream, updateStream, deleteStream, getStreamsUser, grantPermissionsStream, revokePermissionsStream } = require( '../streams/services' )
const { createCommit, createObject, createObjects, getObject, getObjects, getObjectChildren } = require( '../objects/services' )
const { createCommit, createObject, createObjects, getObject, getObjects, getObjectChildren, getObjectChildrenQuery } = require( '../objects/services' )
const sampleObjects = require( './sampleObjectData' )
@@ -154,23 +154,57 @@ describe( 'Objects', ( ) => {
expect( match2.id ).to.equal( objs[ 2 ].id )
} )
let parentObjectId
it( 'Should get object children', async ( ) => {
let objs_1 = createAShitTonOfFuckingObjects( 10000, 'noise__' )
let objs_1 = createManyObjects( 100, 'noise__' )
let ids = await createObjects( objs_1 )
// let objs_2 = createAShitTonOfFuckingObjects( 20000, 'noise_2' )
// let objs_2 = createManyObjects( 20000, 'noise_2' )
// let ids2 = await createObjects( objs_2 )
// let objs_3 = createAShitTonOfFuckingObjects( 50000, 'noise_3' )
// let objs_3 = createManyObjects( 100000, 'noise_3' )
// let ids3 = await createObjects( objs_3 )
console.log( `base id is: ${ids[0]} ` )
console.log( `base id is: ${ids2[0]} ` )
console.log( `base id is: ${ids3[0]} ` )
// let { rows } = await getObjectChildren( { objectId: ids[0], select: ['id', 'name', 'sortValueB'] } )
// let { rows } = await getObjectChildren( { objectId: ids[ 0 ] } )
let limit = 50
let { rows: rows_1, cursor: cursor_1 } = await getObjectChildren( { limit, objectId: ids[ 0 ], select: [ 'nest.mallard', 'test.value', 'test.secondValue', 'nest.arr[0]', 'nest.arr[1]' ] } )
expect( rows_1.length ).to.equal( limit )
expect( rows_1[ 0 ] ).to.be.an( 'object' )
expect( rows_1[ 0 ] ).to.have.property( 'id' )
expect( rows_1[ 0 ] ).to.have.nested.property( 'test.secondValue' )
expect( rows_1[ 0 ] ).to.have.nested.property( 'nest.mallard' )
expect( cursor_1 ).to.be.a( 'string' )
let { rows: rows_2, cursor: cursor_2 } = await getObjectChildren( { limit, objectId: ids[ 0 ], select: [ 'nest.mallard', 'test.value', 'test.secondValue', 'nest.arr[0]', 'nest.arr[1]' ], cursor: cursor_1 } )
expect( rows_2.length ).to.equal( 50 )
expect( rows_2[ 0 ] ).to.be.an( 'object' )
expect( rows_2[ 0 ] ).to.have.property( 'id' )
expect( rows_2[ 0 ] ).to.have.nested.property( 'test.secondValue' )
expect( rows_2[ 0 ] ).to.have.nested.property( 'nest.mallard' )
let { rows, cursor } = await getObjectChildren( { objectId: ids[ 0 ], limit: 1000 } )
expect( rows.length ).to.equal( 100 )
parentObjectId = ids[ 0 ]
} ).timeout( 30000 )
it( 'should query object children', async ( ) => {
// we're assuming the prev test objects exist
let test = await getObjectChildrenQuery( { objectId: parentObjectId, select: [ 'nest.mallard', 'test.value' ] } )
} )
} )
describe( 'Integration (API)', ( ) => {
@@ -256,9 +290,9 @@ describe( 'Objects', ( ) => {
const crypto = require( 'crypto' )
function createAShitTonOfFuckingObjects( shitTon, noise ) {
function createManyObjects( shitTon, noise ) {
shitTon = shitTon || 10000
noise = noise || Math.random() * 100
noise = noise || Math.random( ) * 100
let objs = [ ]
@@ -266,13 +300,21 @@ function createAShitTonOfFuckingObjects( shitTon, noise ) {
objs.push( base )
for ( let i = 0; i < shitTon; i++ ) {
let baby = { name: `mr. ${i}`, noise: noise, sortValueA: i, sortValueB: i * 0.42 * i }
let baby = {
name: `mr. ${i}`,
nest: { duck: true, mallard: 'false', arr: [ i + 42, i, i ] },
test: { value: i, secondValue: 'mallard ' + i % 10 },
objArr: [ { a: i }, { b: i * i }, { c: true } ],
noise: noise,
sortValueA: i,
sortValueB: i * 0.42 * i
}
getAFuckingId( baby )
base.__closure[ baby.id ] = 1
if( i > 1000 )
if ( i > 1000 )
base.__closure[ baby.id ] = i / 1000
objs.push( baby )
}
package-lock.json
Generated
+5
View File
@@ -3392,6 +3392,11 @@
"resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz",
"integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ=="
},
"lodash.set": {
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/lodash.set/-/lodash.set-4.3.2.tgz",
"integrity": "sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM="
},
"lodash.sortby": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz",
package.json
+1
View File
@@ -27,6 +27,7 @@
"graphql-tools": "^4.0.7",
"knex": "^0.20.12",
"lodash.merge": "^4.6.2",
"lodash.set": "^4.3.2",
"lodash.values": "^4.3.0",
"morgan": "^1.10.0",
"morgan-debug": "^2.0.0",
test-queries/closure-fullcount.sql
+8 -7
View File
@@ -1,18 +1,19 @@
with objs as (
SELECT
-- child as id,
id,
serial_id, -- just for reference
"data"
FROM object_children_closure
JOIN objects ON objects.id = child
WHERE parent = '7919a52c017be262ee0daf1844c376d7'
AND "minDepth" < 1000
-- AND (objects."data" -> 'sortValueA')::numeric <= 700
WHERE parent = '89b42e4109f32b20763243d4313e81b5'
-- AND "minDepth" < 1000
AND (jsonb_path_query("data", '$.sortValueA.test.value'))::numeric <= 10
-- AND (objects."data" -> 'sortValueA')::numeric > 100
ORDER BY id
)
SELECT * FROM objs
RIGHT JOIN (SELECT count(*) FROM objs ) c(total_count) ON TRUE
OFFSET 100
LIMIT 200
OFFSET 0
LIMIT 200
-- with "objs" as (select *) select * from "objs" RIGHT JOIN ( SELECT count(*) FROM "objs" ) c(total_count) ON TRUE limit 50