From 426bd3465f2b2ad14aa5aff4749cde5956fdcec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Jedlicska?= Date: Fri, 28 Jul 2023 12:12:07 +0200 Subject: [PATCH] feat(server): user faster string size calculation --- packages/server/modules/core/utils/chunking.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/server/modules/core/utils/chunking.ts b/packages/server/modules/core/utils/chunking.ts index 0a1b0f047..59beab8c1 100644 --- a/packages/server/modules/core/utils/chunking.ts +++ b/packages/server/modules/core/utils/chunking.ts @@ -13,10 +13,15 @@ export class ArgumentError extends BaseError { } } -// Js uses utf16 so string size in byes is length * 2 -export const calculateStringByteSize = (str: string) => new Blob([str]).size +// since we're mostly using this for an artificial limit calculation +// we can live with a somewhat imprecise but fast estimate +// Js uses utf16 so the in memory string size in bytes is length * 2 +// this is just the in memory string size, not the utf-8 encoded byte size +// since our data is mostly ascii characters, its prob safe to use +// string.length is a slight underestimation of the actual size +export const estimateStringByteSize = (str: string) => str.length export const calculateStringMegabyteSize = (str: string) => - calculateStringByteSize(str) / 1_000_000 + estimateStringByteSize(str) / 1_000_000 export const chunkInsertionObjectArray = ({ objects,