test(upload/download): finalised up/down tests & fixed implementation bugs in the process 💀

modules/core/rest/download.js

@@ -44,28 +44,31 @@ module.exports = ( app ) => {
     if ( !simpleText ) gzip.write( '[' )
 
     // helper func to flush the gzip buffer
-    const writeBuffer = ( ) => {
-      console.log( `writing buff ${currentChunkSize}` )
+    const writeBuffer = ( addTrailingComma ) => {
+      // console.log( `writing buff ${currentChunkSize}` )
       if ( simpleText ) {
         gzip.write( chunk )
       } else {
-        gzip.write( `${chunk.join(',')}` )
+        gzip.write( chunk.join( ',' ) )
+        if ( addTrailingComma ){
+          gzip.write( ',' )
+        }
       }
       gzip.flush( )
-      currentChunkSize = 0
       chunk = simpleText ? '' : [ ]
     }
 
     // Populate first object (the "commit")
-    let obj = await getObject( req.params.objectId )
+    let obj = await getObject( { objectId: req.params.objectId } )
     var objString = JSON.stringify( obj )
     if ( simpleText ) {
       chunk += `${obj.id}\t${objString}\n`
     } else {
-      chunk.push( objString + ',' )
+      chunk.push( objString )
     }
-    writeBuffer( )
+    writeBuffer( true )
 
+    let k = 0
     dbStream.on( 'data', row => {
       let data = JSON.stringify( row.data )
       currentChunkSize += Buffer.byteLength( data, 'utf8' )
@@ -75,8 +78,10 @@ module.exports = ( app ) => {
         chunk.push( data )
       }
       if ( currentChunkSize >= maxChunkSize ) {
-        writeBuffer( )
+        currentChunkSize = 0
+        writeBuffer( true )
       }
+      k++
     } )
 
     dbStream.on( 'error', err => {
@@ -85,13 +90,13 @@ module.exports = ( app ) => {
 
     dbStream.on( 'end', ( ) => {
       if ( currentChunkSize !== 0 ) {
-        writeBuffer( )
+        writeBuffer( false )
         if ( !simpleText ) gzip.write( ']' )
       }
       gzip.end( )
-      console.log( 'written end' )
     } )
 
+    // 🚬
     gzip.pipe( res )
   } )
 

modules/core/services/objects.js

@@ -221,8 +221,6 @@ module.exports = {
     q.select( 'data' )
     q.rightJoin( 'objects', 'objects.id', 'object_children_closure.child' )
       .where( knex.raw( 'parent = ?', [ objectId ] ) )
-      // .andWhere( knex.raw( '"minDepth" < ?', [ depth ] ) )
-      // .andWhere( knex.raw( 'id > ?', [ cursor ? cursor : '0' ] ) )
       .orderBy( 'objects.id' )
     return q.stream( )
   },

modules/core/tests/rest.spec.js

@@ -39,7 +39,10 @@ describe( `Upload/Download Routes`, ( ) => {
     testStream.id = await createStream( testStream, userA.id )
   } )
 
-  after( async ( ) => {} )
+  after( async ( ) => {
+    await knex.migrate.rollback( )
+
+  } )
 
   it( 'Should not allow upload requests without an authorization token or valid streamId', async ( ) => {
 
@@ -71,10 +74,11 @@ describe( `Upload/Download Routes`, ( ) => {
   } )
 
   let parentId
+  let numObjs = 5000
 
   it( 'Should properly upload a bunch of objects', async ( ) => {
-    let objBatches = [ createManyObjects( 3000 ), createManyObjects( 3000 ), createManyObjects( 3000 ) ]
-    parentId = objBatches[0][0].id
+    let objBatches = [ createManyObjects( numObjs ), createManyObjects( numObjs ), createManyObjects( numObjs ) ]
+    parentId = objBatches[ 0 ][ 0 ].id
 
     let res =
       await request( expressApp )
@@ -87,14 +91,65 @@ describe( `Upload/Download Routes`, ( ) => {
     expect( res ).to.have.status( 303 )
   } )
 
-  it( 'Should properly download an object, with all its children', async ( ) => {
-    console.log( parentId )
-    let res = await request( expressApp )
-      .get(`/objects/${testStream.id}/${parentId}` )
-      .set( 'Authorization', userA.token )
+  it( 'Should properly download an object, with all its children, into a application/json response', ( done ) => {
+    new Promise( resolve => setTimeout( resolve, 1000 ) ) // avoids race condition
+      .then( ( ) => {
+        let res = request( expressApp )
+          .get( `/objects/${testStream.id}/${parentId}` )
+          .set( 'Authorization', userA.token )
+          .buffer( )
+          .parse( ( res, cb ) => {
+            res.data = ''
+            res.on( 'data', chunk => {
+              res.data += chunk.toString( )
+            } )
+            res.on( 'end', ( ) => {
+              cb( null, res.data )
+            } )
+          } )
+          .end( ( err, res ) => {
+            if ( err ) done( err )
+            try {
+              let o = JSON.parse( res.body )
+              expect( o.length ).to.equal( numObjs + 1 )
+              expect( res ).to.be.json
+              done( )
+            } catch ( err ) {
+              done( err )
+            }
+          } )
+      } )
+  } ).timeout( 5000 )
 
-    console.log( res.status)
-    assert.fail( )
+  it( 'Should properly download an object, with all its children, into a text/plain response', ( done ) => {
+    // new Promise( resolve => setTimeout( resolve, 1000 ) ) // avoids race condition
+    //   .then( ( ) => {
+    let res = request( expressApp )
+      .get( `/objects/${testStream.id}/${parentId}` )
+      .set( 'Authorization', userA.token )
+      .set( 'Accept', 'text/plain' )
+      .buffer( )
+      .parse( ( res, cb ) => {
+        res.data = ''
+        res.on( 'data', chunk => {
+          res.data += chunk.toString( )
+        } )
+        res.on( 'end', ( ) => {
+          cb( null, res.data )
+        } )
+      } )
+      .end( ( err, res ) => {
+        if ( err ) done( err )
+        try {
+          let o = res.body.split( '\n' ).filter( l => l !== '' )
+          expect( o.length ).to.equal( numObjs + 1 )
+          expect( res ).to.be.text
+          done( )
+        } catch ( err ) {
+          done( err )
+        }
+      } )
+    // } )
   } )
 
 } )
@@ -107,19 +162,10 @@ function createManyObjects( amount, noise ) {
 
   let base = { name: 'base bastard 2', noise: noise, __closure: {} }
   objs.push( base )
-  let k = 0
 
   for ( let i = 0; i < amount; i++ ) {
-    let baby = {
-      name: `mr. ${i}`,
-      nest: { duck: i % 2 === 0, mallard: 'falsey', arr: [ i + 42, i, i ] },
-      test: { value: i, secondValue: 'mallard ' + i % 10 },
-      similar: k,
-    }
-
-    if ( i % 3 === 0 ) k++
+    let baby = { name: `mr. ${i}` }
     getId( baby )
-
     base.__closure[ baby.id ] = 1
     objs.push( baby )
   }