Feature: Added diff to @speckle/objectsender (#4646)

* added diffing step to objectsender

* fixed package version

---------

Co-authored-by: Gergő Jedlicska <gergo@jedlicska.com>

packages/objectsender/src/transports/ITransport.ts

@@ -3,8 +3,9 @@ export interface ITransport {
    * Instructs the transport to write this object to its storage layer.
    * @param serialisedObject
    * @param size approximate objects size
+   * @param objectId id of the serialised object
    */
-  write(serialisedObject: string, size: number): Promise<void>
+  write(serialisedObject: string, size: number, objectId: string): Promise<void>
   /**
    * Flushes the buffer ensuring it is persisted to its storage layer.
    */

packages/objectsender/src/transports/ServerTransport.ts

@@ -12,7 +12,7 @@ export type TransportOptions = Partial<{
  * Basic object sender to a speckle server
  */
 export class ServerTransport implements ITransport, IDisposable {
-  #buffer: string[]
+  #buffer: [string, string][]
   #maxSize: number
   #currSize: number
   #serverUrl: string
@@ -38,8 +38,8 @@ export class ServerTransport implements ITransport, IDisposable {
     this.#buffer = []
   }
 
-  async write(serialisedObject: string, size: number) {
-    this.#buffer.push(serialisedObject)
+  async write(serialisedObject: string, size: number, objectId: string) {
+    this.#buffer.push([objectId, serialisedObject])
     this.#currSize += size
     if (this.#currSize < this.#maxSize) return // return fast
     await this.flush() // block until we send objects
@@ -48,8 +48,9 @@ export class ServerTransport implements ITransport, IDisposable {
   async flush() {
     if (this.#buffer.length === 0) return
 
+    const speckleObjects = await this.diff()
     const formData = new FormData()
-    const concat = '[' + this.#buffer.join(',') + ']'
+    const concat = '[' + speckleObjects.join(',') + ']'
     formData.append('object-batch', new Blob([concat], { type: 'application/json' }))
     const url = new URL(`/objects/${this.#projectId}`, this.#serverUrl)
     const res = await retry(
@@ -78,6 +79,31 @@ export class ServerTransport implements ITransport, IDisposable {
     this.#currSize = 0
   }
 
+  async diff() {
+    const objectIds = this.#buffer.map(([id]) => id)
+
+    const url = new URL(`/api/diff/${this.#projectId}`, this.#serverUrl)
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${this.#authToken}`,
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({ objects: JSON.stringify(objectIds) })
+    })
+
+    if (!response.ok) {
+      const data = (await response.json()) as { error: Error }
+      throw new Error(
+        `Unexpected error when sending data. Received ${data.error.message}`
+      )
+    }
+
+    const existingObjects = (await response.json()) as Record<string, boolean>
+
+    return this.#buffer.filter(([id]) => !existingObjects[id]).map(([, value]) => value)
+  }
+
   dispose() {
     this.#buffer = []
   }

packages/objectsender/src/utils/Serializer.ts

@@ -144,7 +144,7 @@ export class Serializer implements IDisposable {
 
     // Pop it in
     if ((detached || root) && this.transport) {
-      await this.transport.write(serializedObject, size)
+      await this.transport.write(serializedObject, size, hash)
     }
 
     // We've reached the end, let's flush
@@ -240,7 +240,7 @@ export class Serializer implements IDisposable {
     const h = this.hashingFunction(s)
     const f = s.substring(0, 1) + `"id":"${h}",` + s.substring(1)
     return {
-      hash: md5(s),
+      hash: h,
       serializedObject: f,
       size: s.length // approx, good enough as we're just limiting artificially batch sizes based on this
     }