diff --git a/packages/objectsender/src/transports/ITransport.ts b/packages/objectsender/src/transports/ITransport.ts index 1c249652e..96c631406 100644 --- a/packages/objectsender/src/transports/ITransport.ts +++ b/packages/objectsender/src/transports/ITransport.ts @@ -3,8 +3,9 @@ export interface ITransport { * Instructs the transport to write this object to its storage layer. * @param serialisedObject * @param size approximate objects size + * @param objectId id of the serialised object */ - write(serialisedObject: string, size: number): Promise + write(serialisedObject: string, size: number, objectId: string): Promise /** * Flushes the buffer ensuring it is persisted to its storage layer. */ diff --git a/packages/objectsender/src/transports/ServerTransport.ts b/packages/objectsender/src/transports/ServerTransport.ts index b74ac7046..ea95b3513 100644 --- a/packages/objectsender/src/transports/ServerTransport.ts +++ b/packages/objectsender/src/transports/ServerTransport.ts @@ -12,7 +12,7 @@ export type TransportOptions = Partial<{ * Basic object sender to a speckle server */ export class ServerTransport implements ITransport, IDisposable { - #buffer: string[] + #buffer: [string, string][] #maxSize: number #currSize: number #serverUrl: string @@ -38,8 +38,8 @@ export class ServerTransport implements ITransport, IDisposable { this.#buffer = [] } - async write(serialisedObject: string, size: number) { - this.#buffer.push(serialisedObject) + async write(serialisedObject: string, size: number, objectId: string) { + this.#buffer.push([objectId, serialisedObject]) this.#currSize += size if (this.#currSize < this.#maxSize) return // return fast await this.flush() // block until we send objects @@ -48,8 +48,9 @@ export class ServerTransport implements ITransport, IDisposable { async flush() { if (this.#buffer.length === 0) return + const speckleObjects = await this.diff() const formData = new FormData() - const concat = '[' + this.#buffer.join(',') + ']' + const concat = '[' + speckleObjects.join(',') + ']' formData.append('object-batch', new Blob([concat], { type: 'application/json' })) const url = new URL(`/objects/${this.#projectId}`, this.#serverUrl) const res = await retry( @@ -78,6 +79,31 @@ export class ServerTransport implements ITransport, IDisposable { this.#currSize = 0 } + async diff() { + const objectIds = this.#buffer.map(([id]) => id) + + const url = new URL(`/api/diff/${this.#projectId}`, this.#serverUrl) + const response = await fetch(url, { + method: 'POST', + headers: { + Authorization: `Bearer ${this.#authToken}`, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ objects: JSON.stringify(objectIds) }) + }) + + if (!response.ok) { + const data = (await response.json()) as { error: Error } + throw new Error( + `Unexpected error when sending data. Received ${data.error.message}` + ) + } + + const existingObjects = (await response.json()) as Record + + return this.#buffer.filter(([id]) => !existingObjects[id]).map(([, value]) => value) + } + dispose() { this.#buffer = [] } diff --git a/packages/objectsender/src/utils/Serializer.ts b/packages/objectsender/src/utils/Serializer.ts index f1c088952..34e2d13a2 100644 --- a/packages/objectsender/src/utils/Serializer.ts +++ b/packages/objectsender/src/utils/Serializer.ts @@ -144,7 +144,7 @@ export class Serializer implements IDisposable { // Pop it in if ((detached || root) && this.transport) { - await this.transport.write(serializedObject, size) + await this.transport.write(serializedObject, size, hash) } // We've reached the end, let's flush @@ -240,7 +240,7 @@ export class Serializer implements IDisposable { const h = this.hashingFunction(s) const f = s.substring(0, 1) + `"id":"${h}",` + s.substring(1) return { - hash: md5(s), + hash: h, serializedObject: f, size: s.length // approx, good enough as we're just limiting artificially batch sizes based on this }