diff --git a/modules/core/migrations/000-core.js b/modules/core/migrations/000-core.js index ce2bde832..832b54750 100644 --- a/modules/core/migrations/000-core.js +++ b/modules/core/migrations/000-core.js @@ -66,6 +66,8 @@ exports.up = async knex => { table.string( 'id' ).primary( ) table.string( 'speckle_type' ).defaultTo( 'Base' ).notNullable( ) table.string( 'applicationId' ) + table.integer( 'totalChildrenCount' ) + table.jsonb( 'totalChildrenCountByDepth' ) table.jsonb( 'data' ) table.string( 'author', 10 ).references( 'id' ).inTable( 'users' ) table.string( 'description' ) @@ -74,6 +76,7 @@ exports.up = async knex => { } ) await knex.raw( 'ALTER TABLE "objects" add column "serial_id" bigserial' ) + await knex.raw( 'CREATE INDEX serial_idx ON objects(serial_id) ' ) // Tree inheritance tracker await knex.schema.createTable( 'object_tree_refs', table => { @@ -81,15 +84,16 @@ exports.up = async knex => { table.string( 'parent' ).index( null, 'HASH' ) table.specificType( 'path', 'ltree' ) } ) + await knex.raw( `CREATE INDEX tree_path_idx ON object_tree_refs USING gist(path)` ) await knex.schema.createTable( 'object_children_closure', table => { - table.string( 'parent' ).notNullable( ) - table.string( 'child' ).notNullable( ) - table.integer( 'minDepth' ).defaultTo( 1 ).notNullable() - // table.index( [ 'parent', 'child' ], 'pc_index' ) + table.string( 'parent' ).notNullable( ).index() + table.string( 'child' ).notNullable( ).index() + table.integer( 'minDepth' ).defaultTo( 1 ).notNullable().index() + table.index( [ 'parent', 'child' ], 'parent_child_index' ) + table.index( [ 'parent', 'minDepth' ], 'full_pcd_index' ) } ) - await knex.raw( `CREATE INDEX tree_path_idx ON object_tree_refs USING gist(path)` ) // creates an enum type for db reference types (branch, tag). await knex.raw( ` diff --git a/modules/core/objects/services.js b/modules/core/objects/services.js index f3e33f28a..a5c1701b4 100644 --- a/modules/core/objects/services.js +++ b/modules/core/objects/services.js @@ -88,15 +88,26 @@ module.exports = { batch.forEach( obj => { let insertionObject = prepInsertionObject( obj ) - + let totalChildrenCountByDepth = {} + let totalChildrenCountGlobal = 0 if ( obj.__closure !== null ) { for ( const prop in obj.__closure ) { closures.push( { parent: insertionObject.id, child: prop, minDepth: obj.__closure[ prop ] } ) + + totalChildrenCountGlobal++ + + if( totalChildrenCountByDepth[ obj.__closure[prop].toString() ] ) + totalChildrenCountByDepth[ obj.__closure[ prop ].toString() ]++ + else + totalChildrenCountByDepth[ obj.__closure[ prop ].toString() ] = 1 } } - delete obj.__tree - delete obj.__closure + insertionObject.totalChildrenCount = totalChildrenCountGlobal + insertionObject.totalChildrenCountByDepth = JSON.stringify( totalChildrenCountByDepth ) + + delete insertionObject.__tree + delete insertionObject.__closure objsToInsert.push( insertionObject ) ids.push( insertionObject.id ) diff --git a/modules/core/tests/objects.spec.js b/modules/core/tests/objects.spec.js index 8e0abb62c..8fa4f3f91 100644 --- a/modules/core/tests/objects.spec.js +++ b/modules/core/tests/objects.spec.js @@ -156,13 +156,19 @@ describe( 'Objects', ( ) => { it( 'Should get object children', async ( ) => { - let nestedBoys = createAShitTonOfFuckingObjects( 300000 ) + let objs_1 = createAShitTonOfFuckingObjects( 10000, 'noise__' ) + let ids = await createObjects( objs_1 ) - let ids = await createObjects( nestedBoys ) + // let objs_2 = createAShitTonOfFuckingObjects( 20000, 'noise_2' ) + // let ids2 = await createObjects( objs_2 ) + + // let objs_3 = createAShitTonOfFuckingObjects( 50000, 'noise_3' ) + // let ids3 = await createObjects( objs_3 ) + console.log( `base id is: ${ids[0]} ` ) + console.log( `base id is: ${ids2[0]} ` ) + console.log( `base id is: ${ids3[0]} ` ) - let res = await getObjectChildren( '0_hash' ) - // console.log( res ) } ).timeout( 30000 ) } ) diff --git a/test-queries/closure-fullcount.sql b/test-queries/closure-fullcount.sql new file mode 100644 index 000000000..e42183379 --- /dev/null +++ b/test-queries/closure-fullcount.sql @@ -0,0 +1,18 @@ +with objs as ( + SELECT +-- child as id, + id, + serial_id, -- just for reference + "data" + FROM object_children_closure + JOIN objects ON objects.id = child + WHERE parent = '7919a52c017be262ee0daf1844c376d7' + AND "minDepth" < 1000 +-- AND (objects."data" -> 'sortValueA')::numeric <= 700 +-- AND (objects."data" -> 'sortValueA')::numeric > 100 + ORDER BY id +) +SELECT * FROM objs +RIGHT JOIN (SELECT count(*) FROM objs ) c(total_count) ON TRUE +OFFSET 100 +LIMIT 200 \ No newline at end of file diff --git a/test-queries/closure-simple-fast.sql b/test-queries/closure-simple-fast.sql new file mode 100644 index 000000000..4e51ba6d2 --- /dev/null +++ b/test-queries/closure-simple-fast.sql @@ -0,0 +1,7 @@ +-- Fast, because? +SELECT child, "data" FROM object_children_closure +RIGHT JOIN objects ON objects.id = child +WHERE parent = '7919a52c017be262ee0daf1844c376d7' +ORDER BY id +OFFSET 0 +LIMIT 20 \ No newline at end of file diff --git a/test-queries/closure-simple-slow.sql b/test-queries/closure-simple-slow.sql new file mode 100644 index 000000000..3cc8996a8 --- /dev/null +++ b/test-queries/closure-simple-slow.sql @@ -0,0 +1,20 @@ +-- SLOW, because? +-- because we were ordering it by the serial_id! +-- sorting by the id (on which we actually do the join) is 10x faster. +-- nice, to say the least. +SELECT child as id, "data" FROM object_children_closure +RIGHT JOIN objects ON objects.id = child +WHERE parent = '509cb0c19594b731214d3ffed2c011df' +-- minDepth is a way to limit asking for objects up to a specific nested depth. +-- this is useful, for example, when we want to get a stream's top level objects only. +-- AND "minDepth" < 1000 +-- better pagination routine: +-- instead of using offset, we use the last item we "saw" as where clause (last seen id) +-- this assumes that we are ordering results by their id. +-- if we would be ordering them by something else, this clause would need to change. +-- AND id > '5a29a1e000d94d8b9f4c6dd767235903' +-- AND (objects."data" -> 'sortValueA')::numeric <= 700 +ORDER BY id +-- ORDER BY serial_id +OFFSET 0 +LIMIT 200 \ No newline at end of file diff --git a/test-queries/closure-two-stage.sql b/test-queries/closure-two-stage.sql new file mode 100644 index 000000000..aa9c35a7e --- /dev/null +++ b/test-queries/closure-two-stage.sql @@ -0,0 +1,19 @@ +WITH ids AS ( + SELECT child FROM object_children_closure + WHERE parent = '94a0a141c211f60c5e3f859baae125e9' + AND "minDepth" < 100 +), +objs AS ( + SELECT + id, + speckle_type, + "data" + FROM ids + JOIN objects ON ids.child = objects.id + WHERE + (objects."data" -> 'sortValueA')::numeric >= 100 +) +SELECT * FROM objs +RIGHT JOIN (SELECT count(*) FROM objs ) c(totalCount) ON TRUE +OFFSET 120 +LIMIT 1000 \ No newline at end of file diff --git a/test-queries/materialised-fullcount.sql b/test-queries/materialised-fullcount.sql new file mode 100644 index 000000000..9c9b58dfb --- /dev/null +++ b/test-queries/materialised-fullcount.sql @@ -0,0 +1,18 @@ +WITH ids AS ( + SELECT DISTINCT unnest( string_to_array( ltree2text( subltree("path", 1, 2) ), '.') ) as obj_id + FROM object_tree_refs + WHERE parent = '0_hash' + ), + objs AS ( + SELECT obj_id as id, speckle_type, "data" + FROM ids + JOIN objects ON ids.obj_id = objects.id + -- WHERE objects."data" @> '{"text": "This is object 1"}' + ORDER BY jsonb_path_query(data, '$.nest.orderMe' ) DESC + ), + childrenCount AS (SELECT count(*) FROM ids), + resultCount AS (SELECT count(*) FROM objs) + SELECT * from objs + RIGHT JOIN (SELECT count(*) FROM objs) d(totalCount) ON TRUE + OFFSET 100 + LIMIT 200 \ No newline at end of file diff --git a/test-queries/materialised-simple-ordinality.sql b/test-queries/materialised-simple-ordinality.sql new file mode 100644 index 000000000..d0b818ede --- /dev/null +++ b/test-queries/materialised-simple-ordinality.sql @@ -0,0 +1,11 @@ + WITH ids as ( + SELECT (subltree("path", 2, 3))::text as obj_id + FROM object_tree_refs + WHERE path ~ '0_hash.*{2}' + ORDER BY id + ) + SELECT id, speckle_type, "data" -> 'nest' -> 'orderMe' + FROM ids + JOIN objects ON obj_id = objects.id WITH ORDINALITY + OFFSET 2 + LIMIT 50 \ No newline at end of file diff --git a/test-queries/materialised-simple.sql b/test-queries/materialised-simple.sql new file mode 100644 index 000000000..849af2100 --- /dev/null +++ b/test-queries/materialised-simple.sql @@ -0,0 +1,10 @@ +WITH ids AS( + SELECT unnest( string_to_array( ltree2text( subltree("path", 1, 3) ), '.') ) as obj_id + FROM object_tree_refs + WHERE parent = '0_hash' +) +SELECT obj_id, speckle_type, "data" +FROM ids +JOIN objects ON ids.obj_id = objects.id +OFFSET 0 +LIMIT 100 \ No newline at end of file