feat(tree): figuring out the best queries for object children

modules/core/migrations/000-core.js

@@ -73,13 +73,21 @@ exports.up = async knex => {
     table.index( [ 'speckle_type' ], 'type_index' )
   } )
 
+  await knex.raw( 'ALTER TABLE "objects" add column "serial_id" bigserial' )
+
   // Tree inheritance tracker
   await knex.schema.createTable( 'object_tree_refs', table => {
     table.increments( 'id' )
-    table.string( 'parent' )
+    table.string( 'parent' ).index( null, 'HASH' )
     table.specificType( 'path', 'ltree' )
   } )
 
+  await knex.schema.createTable( 'object_children', table => {
+    table.string( 'parent' ).notNullable( )
+    table.string( 'child' ).notNullable( )
+    table.index( [ 'parent', 'child' ], 'pc_index' )
+  } )
+
   await knex.raw( `CREATE INDEX tree_path_idx ON object_tree_refs USING gist(path)` )
 
   // creates an enum type for db reference types (branch, tag).
@@ -132,6 +140,7 @@ exports.down = async knex => {
   await knex.schema.dropTableIfExists( 'user_commits' )
   await knex.schema.dropTableIfExists( 'references' )
   await knex.schema.dropTableIfExists( 'object_tree_refs' )
+  await knex.schema.dropTableIfExists( 'object_children' )
   await knex.schema.dropTableIfExists( 'objects' )
   await knex.schema.dropTableIfExists( 'streams' )
   await knex.schema.dropTableIfExists( 'api_tokens' )

modules/core/objects/services.js

@@ -102,6 +102,7 @@ module.exports = {
 
       let t1 = performance.now( )
       debug( `Batch ${index + 1}/${batches.length}: Stored ${objTreeRefs.length + objsToInsert.length} objects in ${t1-t0}ms.` )
+      // console.log( `Batch ${index + 1}/${batches.length}: Stored ${objTreeRefs.length + objsToInsert.length} objects in ${t1-t0}ms.` )
       resolve( )
     } ) )
 
@@ -142,22 +143,51 @@ module.exports = {
         FROM ids 
         JOIN objects ON ids.obj_id = objects.id
         -- WHERE objects."data" @> '{"text": "This is object 1"}'
-        ${
-          orderBy && orderBy.property && orderBy.direction ? ("ORDER BY jsonb_path_query(data, '$." + orderBy.property + "' ) " + orderBy.direction || "ASC" ) : ""
-        }
-      ),
-      childrenCount AS (SELECT count(*) FROM ids),
-      resultCount AS (SELECT count(*) FROM objs)
+        ${ orderBy && orderBy.property && orderBy.direction ? ("ORDER BY jsonb_path_query(data, '$." + orderBy.property + "' ) " + orderBy.direction || "ASC" ) : "ORDER BY obj_id" }
+      )
       SELECT * from objs
       RIGHT JOIN (SELECT count(*) FROM objs) d(totalCount) ON TRUE
       OFFSET ${offset}
       LIMIT ${limit}
     ` )
 
+    let betterQuery = `
+    WITH ids AS(
+      SELECT unnest( string_to_array( ltree2text( subltree("path", 1, 2) ), '.') ) as obj_id
+      FROM object_tree_refs
+    --   WHERE path ~ '0_hash.*{1}'
+      WHERE nlevel(path) = 2
+    ),
+    objs AS(
+      SELECT obj_id, speckle_type, serial_id, 
+      jsonb_path_query(data, '$.text') as "data.text", 
+      jsonb_path_query(data, '$.nest.flag') as "data.nest.flag", 
+      jsonb_path_query(data, '$.nest.what') as "data.nest.what", 
+      jsonb_path_query(data, '$.arr[1]') as "data.arr[1]", 
+      jsonb_path_query(data, '$.arr[2]') as "data.arr[2]", 
+      jsonb_path_query(data, '$.nest.orderMe') as "data.nest.orderMe"
+      FROM ids
+      JOIN objects ON ids.obj_id = objects.id
+    --   WHERE (objects."data" -> 'nest' ->> 'orderMe')::numeric >= 19001
+    --   AND (objects."data"->'nest'->>'what') LIKE '%42%'
+    )
+    SELECT * FROM objs
+    RIGHT JOIN (SELECT count(*) FROM objs ) c(total_count) ON TRUE
+    ORDER BY serial_id desc
+    OFFSET 310
+    LIMIT 1000
+    `
+
     console.log( rawQuery.toString( ) )
 
+    let t0 = performance.now( )
+
     let res = await rawQuery
-    console.log( res.rows )
+
+    let t1 = performance.now( )
+
+
+    console.log( `Found ${res.rows.length} in ${t1-t0}ms.` )
   },
 
   async getObjects( objectIds ) {

modules/core/tests/objects.spec.js

@@ -153,7 +153,7 @@ describe( 'Objects', ( ) => {
     } )
 
     it( 'Should get object children', async ( ) => {
-      let objectCount = 5
+      let objectCount = 10000
       let objs = [ ]
 
       for ( let i = 0; i < objectCount; i++ ) {
@@ -177,12 +177,12 @@ describe( 'Objects', ( ) => {
         if ( i === 0 ) {
           let __tree = [ ]
 
-          for ( let j = 1; j < objectCount - 1; j++ ) {
+          for ( let j = 1; j < objectCount - 2; j++ ) {
             __tree.push( `0_hash.${j}_hash` )
             __tree.push( `0_hash.${j}_hash.${j+1}_hash` )
-            __tree.push( `0_hash.0nasty_hash.0second_nasty.0third_nasty` )
-            if ( j < objectCount - 2 )
-              __tree.push( `0_hash.${j}_hash.${j+1}_hash.${j+2}_hash` )
+            __tree.push( `0_hash.${j}_hash.${j+1}_hash.${j+2}_hash` )
+            // if ( j < objectCount - 2 )
+            //   __tree.push( `0_hash.${j}_hash.${j+1}_hash.${j+2}_hash` )
           }
 
           objs[ i ].__tree = __tree
@@ -191,11 +191,11 @@ describe( 'Objects', ( ) => {
 
       // console.log( objs )
       let ids = await createObjects( objs )
-      console.log( ids )
+      // console.log( ids )
 
       let res = await getObjectChildren( '0_hash' )
       // console.log( res )
-    } )
+    } ).timeout( 30000 )
 
   } )