diff --git a/modules/core/rest/download.js b/modules/core/rest/download.js index 69bdf73ea..1c277d034 100644 --- a/modules/core/rest/download.js +++ b/modules/core/rest/download.js @@ -44,28 +44,31 @@ module.exports = ( app ) => { if ( !simpleText ) gzip.write( '[' ) // helper func to flush the gzip buffer - const writeBuffer = ( ) => { - console.log( `writing buff ${currentChunkSize}` ) + const writeBuffer = ( addTrailingComma ) => { + // console.log( `writing buff ${currentChunkSize}` ) if ( simpleText ) { gzip.write( chunk ) } else { - gzip.write( `${chunk.join(',')}` ) + gzip.write( chunk.join( ',' ) ) + if ( addTrailingComma ){ + gzip.write( ',' ) + } } gzip.flush( ) - currentChunkSize = 0 chunk = simpleText ? '' : [ ] } // Populate first object (the "commit") - let obj = await getObject( req.params.objectId ) + let obj = await getObject( { objectId: req.params.objectId } ) var objString = JSON.stringify( obj ) if ( simpleText ) { chunk += `${obj.id}\t${objString}\n` } else { - chunk.push( objString + ',' ) + chunk.push( objString ) } - writeBuffer( ) + writeBuffer( true ) + let k = 0 dbStream.on( 'data', row => { let data = JSON.stringify( row.data ) currentChunkSize += Buffer.byteLength( data, 'utf8' ) @@ -75,8 +78,10 @@ module.exports = ( app ) => { chunk.push( data ) } if ( currentChunkSize >= maxChunkSize ) { - writeBuffer( ) + currentChunkSize = 0 + writeBuffer( true ) } + k++ } ) dbStream.on( 'error', err => { @@ -85,13 +90,13 @@ module.exports = ( app ) => { dbStream.on( 'end', ( ) => { if ( currentChunkSize !== 0 ) { - writeBuffer( ) + writeBuffer( false ) if ( !simpleText ) gzip.write( ']' ) } gzip.end( ) - console.log( 'written end' ) } ) + // 🚬 gzip.pipe( res ) } ) diff --git a/modules/core/services/objects.js b/modules/core/services/objects.js index 55d9f5fde..10aa6df4f 100644 --- a/modules/core/services/objects.js +++ b/modules/core/services/objects.js @@ -221,8 +221,6 @@ module.exports = { q.select( 'data' ) q.rightJoin( 'objects', 'objects.id', 'object_children_closure.child' ) .where( knex.raw( 'parent = ?', [ objectId ] ) ) - // .andWhere( knex.raw( '"minDepth" < ?', [ depth ] ) ) - // .andWhere( knex.raw( 'id > ?', [ cursor ? cursor : '0' ] ) ) .orderBy( 'objects.id' ) return q.stream( ) }, diff --git a/modules/core/tests/rest.spec.js b/modules/core/tests/rest.spec.js index c66ac6820..24fbe2f40 100644 --- a/modules/core/tests/rest.spec.js +++ b/modules/core/tests/rest.spec.js @@ -39,7 +39,10 @@ describe( `Upload/Download Routes`, ( ) => { testStream.id = await createStream( testStream, userA.id ) } ) - after( async ( ) => {} ) + after( async ( ) => { + await knex.migrate.rollback( ) + + } ) it( 'Should not allow upload requests without an authorization token or valid streamId', async ( ) => { @@ -71,10 +74,11 @@ describe( `Upload/Download Routes`, ( ) => { } ) let parentId + let numObjs = 5000 it( 'Should properly upload a bunch of objects', async ( ) => { - let objBatches = [ createManyObjects( 3000 ), createManyObjects( 3000 ), createManyObjects( 3000 ) ] - parentId = objBatches[0][0].id + let objBatches = [ createManyObjects( numObjs ), createManyObjects( numObjs ), createManyObjects( numObjs ) ] + parentId = objBatches[ 0 ][ 0 ].id let res = await request( expressApp ) @@ -87,14 +91,65 @@ describe( `Upload/Download Routes`, ( ) => { expect( res ).to.have.status( 303 ) } ) - it( 'Should properly download an object, with all its children', async ( ) => { - console.log( parentId ) - let res = await request( expressApp ) - .get(`/objects/${testStream.id}/${parentId}` ) - .set( 'Authorization', userA.token ) + it( 'Should properly download an object, with all its children, into a application/json response', ( done ) => { + new Promise( resolve => setTimeout( resolve, 1000 ) ) // avoids race condition + .then( ( ) => { + let res = request( expressApp ) + .get( `/objects/${testStream.id}/${parentId}` ) + .set( 'Authorization', userA.token ) + .buffer( ) + .parse( ( res, cb ) => { + res.data = '' + res.on( 'data', chunk => { + res.data += chunk.toString( ) + } ) + res.on( 'end', ( ) => { + cb( null, res.data ) + } ) + } ) + .end( ( err, res ) => { + if ( err ) done( err ) + try { + let o = JSON.parse( res.body ) + expect( o.length ).to.equal( numObjs + 1 ) + expect( res ).to.be.json + done( ) + } catch ( err ) { + done( err ) + } + } ) + } ) + } ).timeout( 5000 ) - console.log( res.status) - assert.fail( ) + it( 'Should properly download an object, with all its children, into a text/plain response', ( done ) => { + // new Promise( resolve => setTimeout( resolve, 1000 ) ) // avoids race condition + // .then( ( ) => { + let res = request( expressApp ) + .get( `/objects/${testStream.id}/${parentId}` ) + .set( 'Authorization', userA.token ) + .set( 'Accept', 'text/plain' ) + .buffer( ) + .parse( ( res, cb ) => { + res.data = '' + res.on( 'data', chunk => { + res.data += chunk.toString( ) + } ) + res.on( 'end', ( ) => { + cb( null, res.data ) + } ) + } ) + .end( ( err, res ) => { + if ( err ) done( err ) + try { + let o = res.body.split( '\n' ).filter( l => l !== '' ) + expect( o.length ).to.equal( numObjs + 1 ) + expect( res ).to.be.text + done( ) + } catch ( err ) { + done( err ) + } + } ) + // } ) } ) } ) @@ -107,19 +162,10 @@ function createManyObjects( amount, noise ) { let base = { name: 'base bastard 2', noise: noise, __closure: {} } objs.push( base ) - let k = 0 for ( let i = 0; i < amount; i++ ) { - let baby = { - name: `mr. ${i}`, - nest: { duck: i % 2 === 0, mallard: 'falsey', arr: [ i + 42, i, i ] }, - test: { value: i, secondValue: 'mallard ' + i % 10 }, - similar: k, - } - - if ( i % 3 === 0 ) k++ + let baby = { name: `mr. ${i}` } getId( baby ) - base.__closure[ baby.id ] = 1 objs.push( baby ) }