huanld/speckle-sketchup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: detaching and chunking

Browse Source
This commit is contained in:
izzy lyseggen
2021-09-17 17:40:27 +01:00
parent 5baf7c6749
commit bbca2f15bd
3 changed files with 295 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
speckle_connector/converter/to_speckle.rb
+34 -34
View File
@@ -52,7 +52,7 @@ module SpeckleSystems::SpeckleConnector
name: definition.name,
# i think the base point is always the origin?
basePoint: speckle_point,
geometry: definition.entities.filter_map { |entity| convert_to_speckle(entity) if entity.typename != "Edge" }
"@geometry" => definition.entities.filter_map { |entity| convert_to_speckle(entity) if entity.typename != "Edge" }
}
end
@@ -68,7 +68,7 @@ module SpeckleSystems::SpeckleConnector
renderMaterial: instance.material.nil? ? nil : material_to_speckle(instance.material),
transform: transform_to_speckle(transform),
insertionPoint: speckle_point(origin[0], origin[1], origin[2]),
blockDefinition: component_definition_to_speckle(instance.definition)
"@blockDefinition" => component_definition_to_speckle(instance.definition)
}
end
@@ -94,38 +94,38 @@ module SpeckleSystems::SpeckleConnector
]
end
def mesh_to_speckle(component_def)
vertices = []
faces = []
pt_count = 0
component_def.entities.each do |entity|
next unless entity.typename == "Face"
# def mesh_to_speckle(component_def)
# vertices = []
# faces = []
# pt_count = 0
# component_def.entities.each do |entity|
# next unless entity.typename == "Face"
mesh = entity.mesh
mesh.points.each do |pt|
vertices.push(length_to_speckle(pt[0]), length_to_speckle(pt[1]), length_to_speckle(pt[2]))
end
mesh.polygons.each do |poly|
faces.push(
case poly.count
when 3 then 0 # tris
when 4 then 1 # polys
else
poly.count # ngons
end
)
faces.push(*poly.map { |coord| coord.abs + pt_count })
end
pt_count += mesh.points.count
end
# mesh = entity.mesh
# mesh.points.each do |pt|
# vertices.push(length_to_speckle(pt[0]), length_to_speckle(pt[1]), length_to_speckle(pt[2]))
# end
# mesh.polygons.each do |poly|
# faces.push(
# case poly.count
# when 3 then 0 # tris
# when 4 then 1 # polys
# else
# poly.count # ngons
# end
# )
# faces.push(*poly.map { |coord| coord.abs + pt_count })
# end
# pt_count += mesh.points.count
# end
{
speckle_type: "Objects.Geometry.Mesh",
units: @units,
vertices: vertices,
faces: faces
}
end
# {
# speckle_type: "Objects.Geometry.Mesh",
# units: @units,
# "@vertices" => vertices,
# faces: faces
# }
# end
def face_to_speckle(face)
vertices = []
@@ -151,8 +151,8 @@ module SpeckleSystems::SpeckleConnector
units: @units,
renderMaterial: face.material.nil? ? nil : material_to_speckle(face.material),
bbox: bounds_to_speckle(face.bounds),
vertices: vertices,
faces: faces
"@(31250)vertices" => vertices,
"@(62500)faces" => faces
}
end
ui/src/components/StreamCard.vue
+66 -21
View File
@@ -43,6 +43,8 @@
/*global sketchup*/
import gql from 'graphql-tag'
import { bus } from '../main'
import { BaseObjectSerializer } from '../utils/serialization'
const zlib = require('zlib')
global.convertedFromSketchup = function (streamId, objects) {
bus.$emit('converted-from-sketchup', streamId, objects)
@@ -65,8 +67,8 @@ export default {
bus.$on('converted-from-sketchup', async (streamId, objects) => {
if (streamId != this.stream.id) return
console.log('received objects from sketchup', objects)
await this.createCommit(objects)
console.log('sent to stream: ' + this.stream.id)
})
},
methods: {
@@ -85,22 +87,58 @@ export default {
return
}
let s = new BaseObjectSerializer()
let { hash, serialized } = s.writeJson({ data: objects, speckle_type: 'Base' })
console.log('hash:', hash, 'serialized:', serialized)
console.log('serializer:', s)
console.log('objects:', s.objects)
try {
this.loading = true
let res = await this.$apollo.mutate({
mutation: gql`
mutation ObjectCreate($params: ObjectCreateInput!) {
objectCreate(objectInput: $params)
}
`,
variables: {
params: {
streamId: this.stream.id,
objects: [{ data: objects, speckle_type: 'Base' }]
}
}
})
// let res = await this.$apollo.mutate({
// mutation: gql`
// mutation ObjectCreate($params: ObjectCreateInput!) {
// objectCreate(objectInput: $params)
// }
// `,
// variables: {
// params: {
// streamId: this.stream.id,
// objects: Object.values(s.objects)
// }
// }
// })
// let formData = new FormData()
// formData.append(
// 'batch-1',
// zlib.gzipSync(Buffer.from(JSON.stringify(Object.values(s.objects))))
// )
// let formData = s.batchObjects()
// let token = localStorage.getItem('SpeckleSketchup.AuthToken')
// let res = await fetch(`https://latest.speckle.dev/objects/${this.stream.id}`, {
// method: 'POST',
// headers: { Authorization: 'Bearer ' + token },
// body: formData
// })
// console.log('res:', res)
// if (res.status !== 201) throw `Upload request failed: ${res}`
let batches = s.batchObjects()
for (let batch of batches) {
let res = await this.sendBatch(batch)
console.log(res)
if (res.status !== 201) throw `Upload request failed: ${res}`
}
let commit = {
streamId: this.stream.id,
branchName: 'main',
objectId: hash,
message: 'sent from sketchup',
sourceApplication: 'sketchup',
totalChildrenCount: s.objects[hash].totalChildrenCount
}
console.log('commit:', commit)
await this.$apollo.mutate({
mutation: gql`
mutation CommitCreate($commit: CommitCreateInput!) {
@@ -108,20 +146,27 @@ export default {
}
`,
variables: {
commit: {
streamId: this.stream.id,
branchName: 'main',
objectId: res.data.objectCreate[0],
message: 'sent from sketchup',
sourceApplication: 'sketchup'
}
commit: commit
}
})
console.log('sent to stream: ' + this.stream.id)
this.loading = false
} catch (err) {
this.loading = false
console.log(err)
}
},
async sendBatch(batch) {
let formData = new FormData()
formData.append('batch-1', zlib.gzipSync(Buffer.from(JSON.stringify(batch))))
let token = localStorage.getItem('SpeckleSketchup.AuthToken')
let res = await fetch(`https://latest.speckle.dev/objects/${this.stream.id}`, {
method: 'POST',
headers: { Authorization: 'Bearer ' + token },
body: formData
})
return res
}
}
}
ui/src/utils/serialization.js
+195
View File
@@ -0,0 +1,195 @@
const zlib = require('zlib')
const crypto = require('crypto')
export class BaseObjectSerializer {
constructor(defaultChunkSize = 1000) {
this.defaultChunkSize = defaultChunkSize
this._resetWriter()
}
_resetWriter() {
this.detachLineage = []
this.lineage = []
this.familyTree = {}
this.closureTable = {}
this.objects = {}
}
writeJson(base) {
this._resetWriter()
self.detachLineage = [true]
let { hash, traversed } = this.traverseBase(base)
this.objects[hash] = traversed
let serialized = JSON.stringify(traversed)
return { hash, serialized }
}
traverseBase(base) {
this.lineage.push(crypto.randomBytes(16).toString('hex'))
let traversed = { id: '', speckle_type: base.speckle_type, totalChildrenCount: 0 }
for (let prop in base) {
const val = base[prop]
// ignore nulls and don't pre-populate the id
if (val === null || prop.startsWith('_') || prop == 'id') continue
// don't need to process primitives
if (typeof val !== 'object') {
traversed[prop] = val
continue
}
const detach = prop.startsWith('@')
// 1. chunked arrays
let detachMatch = prop.match(/^@\((\d*)\)/) // chunk syntax
if (Array.isArray(val) && detachMatch) {
const chunkSize = detachMatch[1] !== '' ? parseInt(detachMatch[1]) : this.defaultChunkSize
let chunks = []
let chunk = {
// eslint-disable-next-line camelcase
speckle_type: 'Speckle.Core.Models.DataChunk',
data: []
}
val.forEach((el, count) => {
if (count && count % chunkSize == 0) {
chunks.push(chunk)
chunk = {
// eslint-disable-next-line camelcase
speckle_type: 'Speckle.Core.Models.DataChunk',
data: []
}
}
chunk.data.push(el)
})
if (chunk.data.length !== 0) chunks.push(chunk)
let chunkRefs = []
chunks.forEach((chunk) => {
this.detachLineage.push(detach) // true
let { hash } = this.traverseBase(chunk)
chunkRefs.push(this.detachHelper(hash))
})
traversed[prop.replace(detachMatch[0], '')] = chunkRefs // strip chunk syntax
continue
}
// strip leading '@' for detach (to be removed in the future when we have a way
// to keep track of detachable props to be consistent with sharp and py)
if (detach) prop = prop.substring(1)
// 2. base object
if (val.speckle_type) {
let child = this.traverseValue({ value: val, detach: detach })
traversed[prop] = detach ? this.detachHelper(child.id) : child
} else {
// 3. anything else (dicts, lists)
traversed[prop] = this.traverseValue({ value: val, detach: detach })
}
}
const detached = this.detachLineage.pop()
// add closures and total children count
let closure = {}
const parent = this.lineage.pop()
if (this.familyTree[parent]) {
Object.entries(this.familyTree[parent]).forEach(([ref, depth]) => {
closure[ref] = depth - this.detachLineage.length
})
}
traversed['totalChildrenCount'] = Object.keys(closure).length
const hash = this.getId(traversed)
traversed.id = hash
if (traversed['totalChildrenCount']) {
traversed['__closure'] = this.closureTable[hash] = closure
}
// save obj string if detached
if (detached) this.objects[hash] = traversed
return { hash, traversed }
}
traverseValue({ value, detach = false }) {
// 1. primitives
if (typeof value !== 'object') return value
// 2. arrays
if (Array.isArray(value)) {
if (!detach) return value.map((el) => this.traverseValue({ value: el }))
let detachedList = []
value.forEach((el) => {
if (typeof el === 'object' && el.speckle_type) {
this.detachLineage.push(detach)
let { hash } = this.traverseBase(el)
detachedList.push(this.detachHelper(hash))
} else {
detachedList.push(this.traverseValue({ value: el, detach: detach }))
}
})
return detachedList
}
// 3. dicts
if (!value.speckle_type) return value
// 4. base objects
if (value.speckle_type) {
this.detachLineage.push(detach)
return this.traverseBase(value).traversed
}
throw `Unsupported type '${typeof value}': ${value}`
}
detachHelper(refHash) {
this.lineage.forEach((parent) => {
if (!this.familyTree[parent]) this.familyTree[parent] = {}
if (
!this.familyTree[parent][refHash] ||
this.familyTree[parent][refHash] > this.detachLineage.length
) {
this.familyTree[parent][refHash] = this.detachLineage.length
}
})
return {
referencedId: refHash,
// eslint-disable-next-line camelcase
speckle_type: 'reference'
}
}
getId(obj) {
return crypto.createHash('md5').update(JSON.stringify(obj)).digest('hex')
}
batchObjects(maxBatchSizeMb = 1) {
const maxSize = maxBatchSizeMb * 1000 * 1000
let batches = []
let batch = []
let batchSize = 0
console.log('START batching objects')
let objects = Object.values(this.objects)
objects.forEach((obj) => {
let currObj = JSON.stringify(obj)
if (batchSize + currObj.length < maxSize) {
batch.push(obj)
batchSize += currObj.length
} else {
batches.push(batch)
batch = [currObj]
batchSize = currObj.length
}
})
batches.push(batch)
console.log('batches:', batches)
return batches
}
}