diff --git a/modules/core/migrations/000-core.js b/modules/core/migrations/000-core.js index f76a427a8..100fd7f9d 100644 --- a/modules/core/migrations/000-core.js +++ b/modules/core/migrations/000-core.js @@ -73,13 +73,21 @@ exports.up = async knex => { table.index( [ 'speckle_type' ], 'type_index' ) } ) + await knex.raw( 'ALTER TABLE "objects" add column "serial_id" bigserial' ) + // Tree inheritance tracker await knex.schema.createTable( 'object_tree_refs', table => { table.increments( 'id' ) - table.string( 'parent' ) + table.string( 'parent' ).index( null, 'HASH' ) table.specificType( 'path', 'ltree' ) } ) + await knex.schema.createTable( 'object_children', table => { + table.string( 'parent' ).notNullable( ) + table.string( 'child' ).notNullable( ) + table.index( [ 'parent', 'child' ], 'pc_index' ) + } ) + await knex.raw( `CREATE INDEX tree_path_idx ON object_tree_refs USING gist(path)` ) // creates an enum type for db reference types (branch, tag). @@ -132,6 +140,7 @@ exports.down = async knex => { await knex.schema.dropTableIfExists( 'user_commits' ) await knex.schema.dropTableIfExists( 'references' ) await knex.schema.dropTableIfExists( 'object_tree_refs' ) + await knex.schema.dropTableIfExists( 'object_children' ) await knex.schema.dropTableIfExists( 'objects' ) await knex.schema.dropTableIfExists( 'streams' ) await knex.schema.dropTableIfExists( 'api_tokens' ) diff --git a/modules/core/objects/services.js b/modules/core/objects/services.js index 11cfa4717..03bd3f698 100644 --- a/modules/core/objects/services.js +++ b/modules/core/objects/services.js @@ -102,6 +102,7 @@ module.exports = { let t1 = performance.now( ) debug( `Batch ${index + 1}/${batches.length}: Stored ${objTreeRefs.length + objsToInsert.length} objects in ${t1-t0}ms.` ) + // console.log( `Batch ${index + 1}/${batches.length}: Stored ${objTreeRefs.length + objsToInsert.length} objects in ${t1-t0}ms.` ) resolve( ) } ) ) @@ -142,22 +143,51 @@ module.exports = { FROM ids JOIN objects ON ids.obj_id = objects.id -- WHERE objects."data" @> '{"text": "This is object 1"}' - ${ - orderBy && orderBy.property && orderBy.direction ? ("ORDER BY jsonb_path_query(data, '$." + orderBy.property + "' ) " + orderBy.direction || "ASC" ) : "" - } - ), - childrenCount AS (SELECT count(*) FROM ids), - resultCount AS (SELECT count(*) FROM objs) + ${ orderBy && orderBy.property && orderBy.direction ? ("ORDER BY jsonb_path_query(data, '$." + orderBy.property + "' ) " + orderBy.direction || "ASC" ) : "ORDER BY obj_id" } + ) SELECT * from objs RIGHT JOIN (SELECT count(*) FROM objs) d(totalCount) ON TRUE OFFSET ${offset} LIMIT ${limit} ` ) + let betterQuery = ` + WITH ids AS( + SELECT unnest( string_to_array( ltree2text( subltree("path", 1, 2) ), '.') ) as obj_id + FROM object_tree_refs + -- WHERE path ~ '0_hash.*{1}' + WHERE nlevel(path) = 2 + ), + objs AS( + SELECT obj_id, speckle_type, serial_id, + jsonb_path_query(data, '$.text') as "data.text", + jsonb_path_query(data, '$.nest.flag') as "data.nest.flag", + jsonb_path_query(data, '$.nest.what') as "data.nest.what", + jsonb_path_query(data, '$.arr[1]') as "data.arr[1]", + jsonb_path_query(data, '$.arr[2]') as "data.arr[2]", + jsonb_path_query(data, '$.nest.orderMe') as "data.nest.orderMe" + FROM ids + JOIN objects ON ids.obj_id = objects.id + -- WHERE (objects."data" -> 'nest' ->> 'orderMe')::numeric >= 19001 + -- AND (objects."data"->'nest'->>'what') LIKE '%42%' + ) + SELECT * FROM objs + RIGHT JOIN (SELECT count(*) FROM objs ) c(total_count) ON TRUE + ORDER BY serial_id desc + OFFSET 310 + LIMIT 1000 + ` + console.log( rawQuery.toString( ) ) + let t0 = performance.now( ) + let res = await rawQuery - console.log( res.rows ) + + let t1 = performance.now( ) + + + console.log( `Found ${res.rows.length} in ${t1-t0}ms.` ) }, async getObjects( objectIds ) { diff --git a/modules/core/tests/objects.spec.js b/modules/core/tests/objects.spec.js index 366825fe3..c1cab34b8 100644 --- a/modules/core/tests/objects.spec.js +++ b/modules/core/tests/objects.spec.js @@ -153,7 +153,7 @@ describe( 'Objects', ( ) => { } ) it( 'Should get object children', async ( ) => { - let objectCount = 5 + let objectCount = 10000 let objs = [ ] for ( let i = 0; i < objectCount; i++ ) { @@ -177,12 +177,12 @@ describe( 'Objects', ( ) => { if ( i === 0 ) { let __tree = [ ] - for ( let j = 1; j < objectCount - 1; j++ ) { + for ( let j = 1; j < objectCount - 2; j++ ) { __tree.push( `0_hash.${j}_hash` ) __tree.push( `0_hash.${j}_hash.${j+1}_hash` ) - __tree.push( `0_hash.0nasty_hash.0second_nasty.0third_nasty` ) - if ( j < objectCount - 2 ) - __tree.push( `0_hash.${j}_hash.${j+1}_hash.${j+2}_hash` ) + __tree.push( `0_hash.${j}_hash.${j+1}_hash.${j+2}_hash` ) + // if ( j < objectCount - 2 ) + // __tree.push( `0_hash.${j}_hash.${j+1}_hash.${j+2}_hash` ) } objs[ i ].__tree = __tree @@ -191,11 +191,11 @@ describe( 'Objects', ( ) => { // console.log( objs ) let ids = await createObjects( objs ) - console.log( ids ) + // console.log( ids ) let res = await getObjectChildren( '0_hash' ) // console.log( res ) - } ) + } ).timeout( 30000 ) } )