diff --git a/speckle_connector/converter/to_speckle.rb b/speckle_connector/converter/to_speckle.rb index d5dd186..350ee0f 100644 --- a/speckle_connector/converter/to_speckle.rb +++ b/speckle_connector/converter/to_speckle.rb @@ -52,7 +52,7 @@ module SpeckleSystems::SpeckleConnector name: definition.name, # i think the base point is always the origin? basePoint: speckle_point, - geometry: definition.entities.filter_map { |entity| convert_to_speckle(entity) if entity.typename != "Edge" } + "@geometry" => definition.entities.filter_map { |entity| convert_to_speckle(entity) if entity.typename != "Edge" } } end @@ -68,7 +68,7 @@ module SpeckleSystems::SpeckleConnector renderMaterial: instance.material.nil? ? nil : material_to_speckle(instance.material), transform: transform_to_speckle(transform), insertionPoint: speckle_point(origin[0], origin[1], origin[2]), - blockDefinition: component_definition_to_speckle(instance.definition) + "@blockDefinition" => component_definition_to_speckle(instance.definition) } end @@ -94,38 +94,38 @@ module SpeckleSystems::SpeckleConnector ] end - def mesh_to_speckle(component_def) - vertices = [] - faces = [] - pt_count = 0 - component_def.entities.each do |entity| - next unless entity.typename == "Face" + # def mesh_to_speckle(component_def) + # vertices = [] + # faces = [] + # pt_count = 0 + # component_def.entities.each do |entity| + # next unless entity.typename == "Face" - mesh = entity.mesh - mesh.points.each do |pt| - vertices.push(length_to_speckle(pt[0]), length_to_speckle(pt[1]), length_to_speckle(pt[2])) - end - mesh.polygons.each do |poly| - faces.push( - case poly.count - when 3 then 0 # tris - when 4 then 1 # polys - else - poly.count # ngons - end - ) - faces.push(*poly.map { |coord| coord.abs + pt_count }) - end - pt_count += mesh.points.count - end + # mesh = entity.mesh + # mesh.points.each do |pt| + # vertices.push(length_to_speckle(pt[0]), length_to_speckle(pt[1]), length_to_speckle(pt[2])) + # end + # mesh.polygons.each do |poly| + # faces.push( + # case poly.count + # when 3 then 0 # tris + # when 4 then 1 # polys + # else + # poly.count # ngons + # end + # ) + # faces.push(*poly.map { |coord| coord.abs + pt_count }) + # end + # pt_count += mesh.points.count + # end - { - speckle_type: "Objects.Geometry.Mesh", - units: @units, - vertices: vertices, - faces: faces - } - end + # { + # speckle_type: "Objects.Geometry.Mesh", + # units: @units, + # "@vertices" => vertices, + # faces: faces + # } + # end def face_to_speckle(face) vertices = [] @@ -151,8 +151,8 @@ module SpeckleSystems::SpeckleConnector units: @units, renderMaterial: face.material.nil? ? nil : material_to_speckle(face.material), bbox: bounds_to_speckle(face.bounds), - vertices: vertices, - faces: faces + "@(31250)vertices" => vertices, + "@(62500)faces" => faces } end diff --git a/ui/src/components/StreamCard.vue b/ui/src/components/StreamCard.vue index 259cd5a..2d305f2 100644 --- a/ui/src/components/StreamCard.vue +++ b/ui/src/components/StreamCard.vue @@ -43,6 +43,8 @@ /*global sketchup*/ import gql from 'graphql-tag' import { bus } from '../main' +import { BaseObjectSerializer } from '../utils/serialization' +const zlib = require('zlib') global.convertedFromSketchup = function (streamId, objects) { bus.$emit('converted-from-sketchup', streamId, objects) @@ -65,8 +67,8 @@ export default { bus.$on('converted-from-sketchup', async (streamId, objects) => { if (streamId != this.stream.id) return console.log('received objects from sketchup', objects) + await this.createCommit(objects) - console.log('sent to stream: ' + this.stream.id) }) }, methods: { @@ -85,22 +87,58 @@ export default { return } + let s = new BaseObjectSerializer() + let { hash, serialized } = s.writeJson({ data: objects, speckle_type: 'Base' }) + console.log('hash:', hash, 'serialized:', serialized) + console.log('serializer:', s) + console.log('objects:', s.objects) try { this.loading = true - let res = await this.$apollo.mutate({ - mutation: gql` - mutation ObjectCreate($params: ObjectCreateInput!) { - objectCreate(objectInput: $params) - } - `, - variables: { - params: { - streamId: this.stream.id, - objects: [{ data: objects, speckle_type: 'Base' }] - } - } - }) + // let res = await this.$apollo.mutate({ + // mutation: gql` + // mutation ObjectCreate($params: ObjectCreateInput!) { + // objectCreate(objectInput: $params) + // } + // `, + // variables: { + // params: { + // streamId: this.stream.id, + // objects: Object.values(s.objects) + // } + // } + // }) + // let formData = new FormData() + // formData.append( + // 'batch-1', + // zlib.gzipSync(Buffer.from(JSON.stringify(Object.values(s.objects)))) + // ) + // let formData = s.batchObjects() + + // let token = localStorage.getItem('SpeckleSketchup.AuthToken') + // let res = await fetch(`https://latest.speckle.dev/objects/${this.stream.id}`, { + // method: 'POST', + // headers: { Authorization: 'Bearer ' + token }, + // body: formData + // }) + // console.log('res:', res) + // if (res.status !== 201) throw `Upload request failed: ${res}` + let batches = s.batchObjects() + for (let batch of batches) { + let res = await this.sendBatch(batch) + console.log(res) + if (res.status !== 201) throw `Upload request failed: ${res}` + } + + let commit = { + streamId: this.stream.id, + branchName: 'main', + objectId: hash, + message: 'sent from sketchup', + sourceApplication: 'sketchup', + totalChildrenCount: s.objects[hash].totalChildrenCount + } + console.log('commit:', commit) await this.$apollo.mutate({ mutation: gql` mutation CommitCreate($commit: CommitCreateInput!) { @@ -108,20 +146,27 @@ export default { } `, variables: { - commit: { - streamId: this.stream.id, - branchName: 'main', - objectId: res.data.objectCreate[0], - message: 'sent from sketchup', - sourceApplication: 'sketchup' - } + commit: commit } }) + console.log('sent to stream: ' + this.stream.id) + this.loading = false } catch (err) { this.loading = false console.log(err) } + }, + async sendBatch(batch) { + let formData = new FormData() + formData.append('batch-1', zlib.gzipSync(Buffer.from(JSON.stringify(batch)))) + let token = localStorage.getItem('SpeckleSketchup.AuthToken') + let res = await fetch(`https://latest.speckle.dev/objects/${this.stream.id}`, { + method: 'POST', + headers: { Authorization: 'Bearer ' + token }, + body: formData + }) + return res } } } diff --git a/ui/src/utils/serialization.js b/ui/src/utils/serialization.js new file mode 100644 index 0000000..3dbf198 --- /dev/null +++ b/ui/src/utils/serialization.js @@ -0,0 +1,195 @@ +const zlib = require('zlib') +const crypto = require('crypto') + +export class BaseObjectSerializer { + constructor(defaultChunkSize = 1000) { + this.defaultChunkSize = defaultChunkSize + this._resetWriter() + } + + _resetWriter() { + this.detachLineage = [] + this.lineage = [] + this.familyTree = {} + this.closureTable = {} + this.objects = {} + } + + writeJson(base) { + this._resetWriter() + self.detachLineage = [true] + let { hash, traversed } = this.traverseBase(base) + this.objects[hash] = traversed + let serialized = JSON.stringify(traversed) + return { hash, serialized } + } + + traverseBase(base) { + this.lineage.push(crypto.randomBytes(16).toString('hex')) + + let traversed = { id: '', speckle_type: base.speckle_type, totalChildrenCount: 0 } + for (let prop in base) { + const val = base[prop] + // ignore nulls and don't pre-populate the id + if (val === null || prop.startsWith('_') || prop == 'id') continue + // don't need to process primitives + if (typeof val !== 'object') { + traversed[prop] = val + continue + } + + const detach = prop.startsWith('@') + + // 1. chunked arrays + let detachMatch = prop.match(/^@\((\d*)\)/) // chunk syntax + if (Array.isArray(val) && detachMatch) { + const chunkSize = detachMatch[1] !== '' ? parseInt(detachMatch[1]) : this.defaultChunkSize + let chunks = [] + let chunk = { + // eslint-disable-next-line camelcase + speckle_type: 'Speckle.Core.Models.DataChunk', + data: [] + } + val.forEach((el, count) => { + if (count && count % chunkSize == 0) { + chunks.push(chunk) + chunk = { + // eslint-disable-next-line camelcase + speckle_type: 'Speckle.Core.Models.DataChunk', + data: [] + } + } + + chunk.data.push(el) + }) + if (chunk.data.length !== 0) chunks.push(chunk) + + let chunkRefs = [] + chunks.forEach((chunk) => { + this.detachLineage.push(detach) // true + let { hash } = this.traverseBase(chunk) + chunkRefs.push(this.detachHelper(hash)) + }) + + traversed[prop.replace(detachMatch[0], '')] = chunkRefs // strip chunk syntax + continue + } + + // strip leading '@' for detach (to be removed in the future when we have a way + // to keep track of detachable props to be consistent with sharp and py) + if (detach) prop = prop.substring(1) + // 2. base object + if (val.speckle_type) { + let child = this.traverseValue({ value: val, detach: detach }) + traversed[prop] = detach ? this.detachHelper(child.id) : child + } else { + // 3. anything else (dicts, lists) + traversed[prop] = this.traverseValue({ value: val, detach: detach }) + } + } + + const detached = this.detachLineage.pop() + + // add closures and total children count + let closure = {} + const parent = this.lineage.pop() + if (this.familyTree[parent]) { + Object.entries(this.familyTree[parent]).forEach(([ref, depth]) => { + closure[ref] = depth - this.detachLineage.length + }) + } + + traversed['totalChildrenCount'] = Object.keys(closure).length + + const hash = this.getId(traversed) + + traversed.id = hash + if (traversed['totalChildrenCount']) { + traversed['__closure'] = this.closureTable[hash] = closure + } + + // save obj string if detached + if (detached) this.objects[hash] = traversed + + return { hash, traversed } + } + + traverseValue({ value, detach = false }) { + // 1. primitives + if (typeof value !== 'object') return value + + // 2. arrays + if (Array.isArray(value)) { + if (!detach) return value.map((el) => this.traverseValue({ value: el })) + + let detachedList = [] + value.forEach((el) => { + if (typeof el === 'object' && el.speckle_type) { + this.detachLineage.push(detach) + let { hash } = this.traverseBase(el) + detachedList.push(this.detachHelper(hash)) + } else { + detachedList.push(this.traverseValue({ value: el, detach: detach })) + } + }) + return detachedList + } + + // 3. dicts + if (!value.speckle_type) return value + + // 4. base objects + if (value.speckle_type) { + this.detachLineage.push(detach) + return this.traverseBase(value).traversed + } + + throw `Unsupported type '${typeof value}': ${value}` + } + + detachHelper(refHash) { + this.lineage.forEach((parent) => { + if (!this.familyTree[parent]) this.familyTree[parent] = {} + if ( + !this.familyTree[parent][refHash] || + this.familyTree[parent][refHash] > this.detachLineage.length + ) { + this.familyTree[parent][refHash] = this.detachLineage.length + } + }) + return { + referencedId: refHash, + // eslint-disable-next-line camelcase + speckle_type: 'reference' + } + } + + getId(obj) { + return crypto.createHash('md5').update(JSON.stringify(obj)).digest('hex') + } + + batchObjects(maxBatchSizeMb = 1) { + const maxSize = maxBatchSizeMb * 1000 * 1000 + let batches = [] + let batch = [] + let batchSize = 0 + console.log('START batching objects') + let objects = Object.values(this.objects) + objects.forEach((obj) => { + let currObj = JSON.stringify(obj) + if (batchSize + currObj.length < maxSize) { + batch.push(obj) + batchSize += currObj.length + } else { + batches.push(batch) + batch = [currObj] + batchSize = currObj.length + } + }) + batches.push(batch) + + console.log('batches:', batches) + + return batches + } +}