Merge branch 'main' into oguzhan/acc-connector

packages/ifc-import-service/pyproject.toml

@@ -5,44 +5,40 @@ description = "Speckle IFC importer worker app"
 readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
-  "asyncpg>=0.30.0",
-  "typed-settings>=24.5.0",
-  "pydantic>=2.11.7",
-  "python-dotenv>=1.0.0",
-  "structlog>=25.4.0",
-  "structlog-to-seq>=21.0.0",
-  "specklepy[speckleifc]>=3.0.4.dev8",
+    "asyncpg>=0.30.0",
+    "typed-settings>=24.5.0",
+    "pydantic>=2.11.7",
+    "python-dotenv>=1.0.0",
+    "structlog>=25.4.0",
+    "structlog-to-seq>=21.0.0",
+    "specklepy[speckleifc]>=3.0.4.dev8",
+    "colorful>=0.5.7",
 ]
 
 [dependency-groups]
-dev = [
-  "asyncpg-stubs>=0.30.2",
-  "colorama>=0.4.6",
-  "colorful>=0.5.7",
-  "ruff>=0.12.2",
-]
+dev = ["asyncpg-stubs>=0.30.2", "colorama>=0.4.6", "ruff>=0.12.2"]
 
 [tool.ruff]
 exclude = [".venv", "**/*.yml"]
 
 [tool.ruff.lint]
 select = [
-  "A",
-  # pycodestyle
-  "E",
-  # Pyflakes
-  "F",
-  # pyupgrade
-  "UP",
-  # flake8-bugbear
-  "B",
-  # flake8-simplify
-  "SIM",
-  # isort
-  "I",
-  # PEP8 naming
-  "N",
-  "ASYNC",
+    "A",
+    # pycodestyle
+    "E",
+    # Pyflakes
+    "F",
+    # pyupgrade
+    "UP",
+    # flake8-bugbear
+    "B",
+    # flake8-simplify
+    "SIM",
+    # isort
+    "I",
+    # PEP8 naming
+    "N",
+    "ASYNC",
 ]
 
 [build-system]

packages/ifc-import-service/src/ifc_importer/domain.py

@@ -31,6 +31,7 @@ class JobStatus(StrEnum):
     """Status enumeration for the job."""
 
     QUEUED = "queued"
+    PROCESSING = "processing"
     SUCCEEDED = "succeeded"
     FAILED = "failed"
 

packages/ifc-import-service/src/ifc_importer/job_processor.py

@@ -71,96 +71,100 @@ async def job_processor(logger: structlog.stdlib.BoundLogger):
     parser = "speckle_ifc"
     connection = await setup_connection()
     while True:
-        async with connection.transaction():
-            job = await get_next_job(connection)
-            if not job:
-                logger.info("no job found")
-                await asyncio.sleep(IDLE_TIMEOUT)
-                continue
+        job = await get_next_job(connection)
+        if not job:
+            logger.info("no job found")
+            await asyncio.sleep(IDLE_TIMEOUT)
+            continue
 
-            start = time.time()
-            speckle_client = setup_client(job.payload)
+        start = time.time()
+        speckle_client = setup_client(job.payload)
 
-            job_id = job.id
-            job_status = JobStatus.QUEUED
-            ex: Exception | None = None
-            attempt = job.attempt + 1
+        job_id = job.id
+        job_status = JobStatus.QUEUED
+        ex: Exception | None = None
+        attempt = job.attempt
+        version_id: str | None = None
 
-            version_id: str | None = None
-
-            try:
-                logger = logger.bind(job_id=job_id, project_id=job.payload.project_id)
-                logger.info("starting job")
-                handler = job_handler(speckle_client, job.payload, logger)
-                # this will raise a TimeoutError if handler does not complete in time
-                version, download_duration, parse_duration = await asyncio.wait_for(
-                    handler, timeout=job.payload.time_out_seconds
-                )
-                version_id = version.id
-
-                duration = time.time() - start
-                logger.info(
-                    "Finished job after {duration} created version {version_id}",
-                    duration=duration,
-                    version_id=version_id,
+        try:
+            if attempt > job.max_attempt:
+                # something went wrong, it should have been marked as failed
+                raise Exception(
+                    "Unhandled error silently failed the job multiple times"
                 )
 
-                _ = speckle_client.file_import.finish_file_import_job(
-                    FileImportSuccessInput(
-                        project_id=job.payload.project_id,
-                        # for some reason, the blob id identifies the job here
-                        job_id=job.payload.blob_id,
-                        result=FileImportResult(
-                            parser=parser,
-                            version_id=version_id,
-                            download_duration_seconds=download_duration,
-                            duration_seconds=duration,
-                            parse_duration_seconds=parse_duration,
-                        ),
-                    )
+            logger = logger.bind(job_id=job_id, project_id=job.payload.project_id)
+            logger.info("starting job")
+            handler = job_handler(speckle_client, job.payload, logger)
+            # this will raise a TimeoutError if handler does not complete in time
+            version, download_duration, parse_duration = await asyncio.wait_for(
+                handler, timeout=job.payload.time_out_seconds
+            )
+            version_id = version.id
+
+            duration = time.time() - start
+            logger.info(
+                "Finished job after {duration} created version {version_id}",
+                duration=duration,
+                version_id=version_id,
+            )
+
+            _ = speckle_client.file_import.finish_file_import_job(
+                FileImportSuccessInput(
+                    project_id=job.payload.project_id,
+                    # for some reason, the blob id identifies the job here
+                    job_id=job.payload.blob_id,
+                    result=FileImportResult(
+                        parser=parser,
+                        version_id=version_id,
+                        download_duration_seconds=download_duration,
+                        duration_seconds=duration,
+                        parse_duration_seconds=parse_duration,
+                    ),
                 )
+            )
 
-                job_status = JobStatus.SUCCEEDED
+            job_status = JobStatus.SUCCEEDED
 
-            except TimeoutError as te:
-                # if it times out we allow re-queueing until it reaches max tries
-                ex = te
-                if attempt >= job.max_attempt:
-                    job_status = JobStatus.FAILED
-                else:
-                    job_status = JobStatus.QUEUED
-
-            # raised if the task is canceled
-            except Exception as e:
-                #
-                ex = e
+        except TimeoutError as te:
+            # if it times out we allow re-queueing until it reaches max tries
+            ex = te
+            if attempt >= job.max_attempt:
                 job_status = JobStatus.FAILED
-            finally:
-                if job_status == JobStatus.FAILED:
-                    # we should be reporting the failure to the server
-                    logger.error("job processing failed", exc_info=ex)
-                    try:
-                        _ = speckle_client.file_import.finish_file_import_job(
-                            FileImportErrorInput(
-                                project_id=job.payload.project_id,
-                                # for some reason, the blob id identifies the job here
-                                job_id=job.payload.blob_id,
-                                # job_id=job_id,
-                                reason=str(ex),
-                                result=FileImportResult(
-                                    parser=parser,
-                                    version_id=None,
-                                    download_duration_seconds=0,
-                                    duration_seconds=time.time() - start,
-                                    parse_duration_seconds=0,
-                                ),
-                            )
+            else:
+                job_status = JobStatus.QUEUED
+
+        # raised if the task is canceled
+        except Exception as e:
+            #
+            ex = e
+            job_status = JobStatus.FAILED
+        finally:
+            if job_status == JobStatus.FAILED:
+                # we should be reporting the failure to the server
+                logger.error("job processing failed", exc_info=ex)
+                try:
+                    _ = speckle_client.file_import.finish_file_import_job(
+                        FileImportErrorInput(
+                            project_id=job.payload.project_id,
+                            # for some reason, the blob id identifies the job here
+                            job_id=job.payload.blob_id,
+                            # job_id=job_id,
+                            reason=str(ex),
+                            result=FileImportResult(
+                                parser=parser,
+                                version_id=None,
+                                download_duration_seconds=0,
+                                duration_seconds=time.time() - start,
+                                parse_duration_seconds=0,
+                            ),
                         )
-                    # if the reporting of the failure does not succeed, we're requeueing
-                    # unless we've reached the max attempts
-                    except Exception as ex:
-                        if attempt >= job.max_attempt:
-                            job_status = JobStatus.FAILED
-                        else:
-                            job_status = JobStatus.QUEUED
-                await set_job_status(connection, job_id, job_status, attempt)
+                    )
+                # if the reporting of the failure does not succeed, we're requeueing
+                # unless we've reached the max attempts
+                except Exception as ex:
+                    if attempt >= job.max_attempt:
+                        job_status = JobStatus.FAILED
+                    else:
+                        job_status = JobStatus.QUEUED
+            await set_job_status(connection, job_id, job_status)

packages/ifc-import-service/src/ifc_importer/repository.py

@@ -22,12 +22,32 @@ async def get_next_job(connection: Connection) -> FileimportJob | None:
 
     job = await connection.fetchrow(
         """
-        SELECT * FROM background_jobs
-        WHERE payload ->> 'fileType' = 'ifc' AND status = $1 AND attempt < "maxAttempt"
-        ORDER BY "createdAt"
-        FOR UPDATE SKIP LOCKED
-        LIMIT 1
+        WITH next_job AS (
+            UPDATE background_jobs
+            SET
+                "attempt" = "attempt" + 1,
+                "status" = $1,
+                "updatedAt" = NOW()
+            WHERE id = (
+                SELECT id FROM background_jobs
+                WHERE ( --queued job
+                    payload ->> 'fileType' = 'ifc'
+                    AND status = $2
+                )
+                OR ( --timed job left on processing state
+                    payload ->> 'fileType' = 'ifc'
+                    AND status = $1
+                    AND "updatedAt" < NOW() - ("timeoutMs" * interval '1 millisecond')
+                )
+                ORDER BY "createdAt"
+                FOR UPDATE SKIP LOCKED
+                LIMIT 1
+            )
+            RETURNING *
+        )
+        SELECT * FROM next_job;
         """,
+        JobStatus.PROCESSING.value,
         JobStatus.QUEUED.value,
     )
     if not job:
@@ -36,16 +56,15 @@ async def get_next_job(connection: Connection) -> FileimportJob | None:
 
 
 async def set_job_status(
-    connection: Connection, job_id: str, job_status: JobStatus, attempt: int
+    connection: Connection, job_id: str, job_status: JobStatus
 ) -> None:
-    print(f"updating job: {job_id}'s status to {job_status}, with attempt: {attempt}")
+    print(f"updating job: {job_id}'s status to {job_status}")
     _ = await connection.execute(
         """
         UPDATE background_jobs
-        SET status = $1, "updatedAt" = NOW(), attempt = $3
+        SET status = $1, "updatedAt" = NOW()
         WHERE id = $2
         """,
         job_status.value,
         job_id,
-        attempt,
     )

packages/ifc-import-service/uv.lock

@@ -251,6 +251,7 @@ version = "0.1.0"
 source = { editable = "." }
 dependencies = [
     { name = "asyncpg" },
+    { name = "colorful" },
     { name = "pydantic" },
     { name = "python-dotenv" },
     { name = "specklepy", extra = ["speckleifc"] },
@@ -263,13 +264,13 @@ dependencies = [
 dev = [
     { name = "asyncpg-stubs" },
     { name = "colorama" },
-    { name = "colorful" },
     { name = "ruff" },
 ]
 
 [package.metadata]
 requires-dist = [
     { name = "asyncpg", specifier = ">=0.30.0" },
+    { name = "colorful", specifier = ">=0.5.7" },
     { name = "pydantic", specifier = ">=2.11.7" },
     { name = "python-dotenv", specifier = ">=1.0.0" },
     { name = "specklepy", extras = ["speckleifc"], specifier = ">=3.0.4.dev8" },
@@ -282,7 +283,6 @@ requires-dist = [
 dev = [
     { name = "asyncpg-stubs", specifier = ">=0.30.2" },
     { name = "colorama", specifier = ">=0.4.6" },
-    { name = "colorful", specifier = ">=0.5.7" },
     { name = "ruff", specifier = ">=0.12.2" },
 ]
 

packages/monitor-deployment/src/observability/metrics/dbMaxPerparedTransactions.ts

@@ -0,0 +1,33 @@
+import prometheusClient from 'prom-client'
+import { join } from 'lodash-es'
+import type { MetricInitializer } from '@/observability/types.js'
+
+export const init: MetricInitializer = (config) => {
+  const { labelNames, namePrefix, logger } = config
+  const metric = new prometheusClient.Gauge({
+    name: join([namePrefix, 'db', 'max_prepared_transactions'], '_'),
+    help: 'Configured value of max_prepared_transactions for the Postgres database',
+    labelNames: ['region', ...labelNames]
+  })
+  return async (params) => {
+    const { dbClients, labels } = params
+    await Promise.all(
+      dbClients.map(async ({ client, regionKey }) => {
+        const queryResults = await client.raw<{
+          rows: [{ max_prepared_transactions: string }]
+        }>(`SHOW max_prepared_transactions;`)
+        if (!queryResults.rows.length) {
+          logger.error(
+            { region: regionKey },
+            "No max_prepared_transactions found for region '{region}'. This is odd."
+          )
+          return
+        }
+        metric.set(
+          { ...labels, region: regionKey },
+          parseInt(queryResults.rows[0].max_prepared_transactions)
+        )
+      })
+    )
+  }
+}

packages/monitor-deployment/src/observability/metrics/dbPreparedTransactions.ts

@@ -0,0 +1,33 @@
+import prometheusClient from 'prom-client'
+import { join } from 'lodash-es'
+import type { MetricInitializer } from '@/observability/types.js'
+
+export const init: MetricInitializer = (config) => {
+  const { labelNames, namePrefix, logger } = config
+  const dbWorkers = new prometheusClient.Gauge({
+    name: join([namePrefix, 'db_prepared_transactions'], '_'),
+    help: 'Number of prepared transactions',
+    labelNames: ['region', ...labelNames]
+  })
+  return async (params) => {
+    const { dbClients, labels } = params
+    await Promise.all(
+      dbClients.map(async ({ client, regionKey }) => {
+        const connectionResults = await client.raw<{
+          rows: [{ prepared_transaction_count: string }]
+        }>(`SELECT COUNT(*) AS prepared_transaction_count FROM pg_prepared_xacts;`)
+        if (!connectionResults.rows.length) {
+          logger.error(
+            { region: regionKey },
+            "No prepared transactions found for region '{region}'. This is odd."
+          )
+          return
+        }
+        dbWorkers.set(
+          { ...labels, region: regionKey },
+          parseInt(connectionResults.rows[0].prepared_transaction_count)
+        )
+      })
+    )
+  }
+}

packages/monitor-deployment/src/observability/prometheusMetrics.ts

@@ -6,10 +6,12 @@ import { Counter, Histogram, Registry } from 'prom-client'
 import prometheusClient from 'prom-client'
 import { init as commits } from '@/observability/metrics/commits.js'
 import { init as dbMaxLogicalReplicationWorkers } from '@/observability/metrics/dbMaxLogicalReplicationWorkers.js'
+import { init as dbMaxPerparedTransactions } from '@/observability/metrics/dbMaxPerparedTransactions.js'
 import { init as dbMaxReplicationSlots } from '@/observability/metrics/dbMaxReplicationSlots.js'
 import { init as dbMaxSyncWorkersPerSubscription } from '@/observability/metrics/dbMaxSyncWorkersPerSubscription.js'
 import { init as dbMaxWalSenders } from '@/observability/metrics/dbMaxWalSenders.js'
 import { init as dbMaxWorkerProcesses } from '@/observability/metrics/dbMaxWorkerProcesses.js'
+import { init as dbPreparedTransactions } from '@/observability/metrics/dbPreparedTransactions.js'
 import { init as dbSize } from '@/observability/metrics/dbSize.js'
 import { init as dbWalLevel } from '@/observability/metrics/dbWalLevel.js'
 import { init as dbWorkers } from '@/observability/metrics/dbWorkers.js'
@@ -62,10 +64,12 @@ function initMonitoringMetrics(params: {
   const metricsToInitialize = [
     commits,
     dbMaxLogicalReplicationWorkers,
+    dbMaxPerparedTransactions,
     dbMaxReplicationSlots,
     dbMaxSyncWorkersPerSubscription,
     dbMaxWalSenders,
     dbMaxWorkerProcesses,
+    dbPreparedTransactions,
     dbWalLevel,
     dbSize,
     dbWorkers,

packages/server/modules/backgroundjobs/index.ts

@@ -4,7 +4,8 @@ import { moduleLogger } from '@/observability/logging'
 const backgroundJobsModule: SpeckleModule = {
   async init() {
     moduleLogger.info('🛠️  Init backgroundjobs module')
-  }
+  },
+  async shutdown() {}
 }
 
 export default backgroundJobsModule

packages/server/modules/backgroundjobs/repositories.ts

@@ -1,12 +1,11 @@
 import type { Knex } from 'knex'
-import type {
-  BackgroundJob,
-  BackgroundJobPayload,
-  GetBackgroundJob,
-  GetBackgroundJobCount,
-  StoreBackgroundJob
+import {
+  type BackgroundJob,
+  type BackgroundJobPayload,
+  type GetBackgroundJob,
+  type GetBackgroundJobCount,
+  type StoreBackgroundJob
 } from '@/modules/backgroundjobs/domain'
-import { BackgroundJobStatus } from '@/modules/backgroundjobs/domain'
 import { buildTableHelper } from '@/modules/core/dbSchema'
 
 export const BackgroundJobs = buildTableHelper('background_jobs', [
@@ -54,22 +53,8 @@ export const getBackgroundJobCountFactory =
   async ({ status, jobType, minAttempts }) => {
     const q = tables.backgroundJobs(db).select(BackgroundJobs.col.id)
 
-    // using less restrictive lock to check locked jobs
-    if (status === BackgroundJobStatus.Processing) {
-      q.whereNotExists(function () {
-        const subquery = this.from(BackgroundJobs.name)
-          .select(BackgroundJobs.col.id)
-          .whereRaw('id = background_jobs.id')
-          .where({ jobType })
-
-        if (minAttempts) {
-          q.andWhere(BackgroundJobs.col.attempt, '>=', minAttempts)
-        }
-
-        subquery.forKeyShare().skipLocked()
-      })
-    } else {
-      q.where({ status }).forKeyShare().skipLocked()
+    if (status) {
+      q.where({ status })
     }
 
     if (minAttempts) {

packages/server/modules/backgroundjobs/services/create.ts

@@ -7,7 +7,7 @@ import type {
 import { BackgroundJobStatus } from '@/modules/backgroundjobs/domain'
 import cryptoRandomString from 'crypto-random-string'
 
-export const scheduleBackgroundJobFactory = <T extends BackgroundJobPayload>({
+export const createBackgroundJobFactory = <T extends BackgroundJobPayload>({
   storeBackgroundJob,
   jobConfig
 }: {

packages/server/modules/backgroundjobs/tests/integration/repositories.spec.ts

@@ -1,4 +1,4 @@
-import knex, { db } from '@/db/knex'
+import { db } from '@/db/knex'
 import {
   storeBackgroundJobFactory,
   getBackgroundJobFactory,
@@ -135,13 +135,10 @@ describe('Background Jobs repositories @backgroundjobs', () => {
     it('is able to count locked jobs', async () => {
       const job = createTestJob({
         jobType: 'fileImport',
-        status: BackgroundJobStatus.Queued
+        status: BackgroundJobStatus.Processing
       })
       await storeBackgroundJob({ job })
 
-      const trx = await knex.transaction()
-      await trx().from(BackgroundJobs.name).where({ id: job.id }).forUpdate().first() // acquire lock
-
       const [processingCount, queuedCount] = await Promise.all([
         getBackgroundJobCount({ status: 'processing', jobType: 'fileImport' }),
         getBackgroundJobCount({ status: 'queued', jobType: 'fileImport' })
@@ -149,8 +146,6 @@ describe('Background Jobs repositories @backgroundjobs', () => {
 
       expect(processingCount).to.equal(1)
       expect(queuedCount).to.equal(0)
-
-      await trx.commit()
     })
 
     it('filters by min attempts', async () => {

packages/server/modules/backgroundjobs/tests/unit/services.spec.ts

@@ -1,5 +1,5 @@
 import { expect } from 'chai'
-import { scheduleBackgroundJobFactory } from '@/modules/backgroundjobs/services'
+import { createBackgroundJobFactory } from '@/modules/backgroundjobs/services/create'
 import type {
   BackgroundJob,
   BackgroundJobConfig,
@@ -41,12 +41,12 @@ describe('scheduleBackgroundJobFactory', () => {
 
   describe('when called with valid parameters', () => {
     it('should create and store a background job with correct properties', async () => {
-      const scheduleBackgroundJob = scheduleBackgroundJobFactory({
+      const createBackgroundJob = createBackgroundJobFactory({
         storeBackgroundJob: mockStoreBackgroundJob,
         jobConfig: mockJobConfig
       })
 
-      const result = await scheduleBackgroundJob({ jobPayload: mockJobPayload })
+      const result = await createBackgroundJob({ jobPayload: mockJobPayload })
 
       expect(storeBackgroundJobCalled).to.be.true
       expect(storedJob).to.not.be.null
@@ -54,58 +54,58 @@ describe('scheduleBackgroundJobFactory', () => {
     })
 
     it('should generate a unique job ID', async () => {
-      const scheduleBackgroundJob = scheduleBackgroundJobFactory({
+      const createBackgroundJob = createBackgroundJobFactory({
         storeBackgroundJob: mockStoreBackgroundJob,
         jobConfig: mockJobConfig
       })
 
-      const result = await scheduleBackgroundJob({ jobPayload: mockJobPayload })
+      const result = await createBackgroundJob({ jobPayload: mockJobPayload })
 
       expect(result.id).to.be.a('string')
       expect(result.id).to.have.length(10)
     })
 
     it('should set job status to Queued', async () => {
-      const scheduleBackgroundJob = scheduleBackgroundJobFactory({
+      const createBackgroundJob = createBackgroundJobFactory({
         storeBackgroundJob: mockStoreBackgroundJob,
         jobConfig: mockJobConfig
       })
 
-      const result = await scheduleBackgroundJob({ jobPayload: mockJobPayload })
+      const result = await createBackgroundJob({ jobPayload: mockJobPayload })
 
       expect(result.status).to.equal(BackgroundJobStatus.Queued)
     })
 
     it('should set attempt to 0', async () => {
-      const scheduleBackgroundJob = scheduleBackgroundJobFactory({
+      const createBackgroundJob = createBackgroundJobFactory({
         storeBackgroundJob: mockStoreBackgroundJob,
         jobConfig: mockJobConfig
       })
 
-      const result = await scheduleBackgroundJob({ jobPayload: mockJobPayload })
+      const result = await createBackgroundJob({ jobPayload: mockJobPayload })
 
       expect(result.attempt).to.equal(0)
     })
 
     it('should include job config properties', async () => {
-      const scheduleBackgroundJob = scheduleBackgroundJobFactory({
+      const createBackgroundJob = createBackgroundJobFactory({
         storeBackgroundJob: mockStoreBackgroundJob,
         jobConfig: mockJobConfig
       })
 
-      const result = await scheduleBackgroundJob({ jobPayload: mockJobPayload })
+      const result = await createBackgroundJob({ jobPayload: mockJobPayload })
 
       expect(result.maxAttempt).to.equal(mockJobConfig.maxAttempt)
       expect(result.timeoutMs).to.equal(mockJobConfig.timeoutMs)
     })
 
     it('should preserve job payload', async () => {
-      const scheduleBackgroundJob = scheduleBackgroundJobFactory({
+      const createBackgroundJob = createBackgroundJobFactory({
         storeBackgroundJob: mockStoreBackgroundJob,
         jobConfig: mockJobConfig
       })
 
-      const result = await scheduleBackgroundJob({ jobPayload: mockJobPayload })
+      const result = await createBackgroundJob({ jobPayload: mockJobPayload })
 
       expect(result.payload).to.deep.equal(mockJobPayload)
       expect(result.jobType).to.equal(mockJobPayload.jobType)
@@ -114,12 +114,12 @@ describe('scheduleBackgroundJobFactory', () => {
     it('should set createdAt and updatedAt timestamps', async () => {
       const beforeTest = new Date()
 
-      const scheduleBackgroundJob = scheduleBackgroundJobFactory({
+      const createBackgroundJob = createBackgroundJobFactory({
         storeBackgroundJob: mockStoreBackgroundJob,
         jobConfig: mockJobConfig
       })
 
-      const result = await scheduleBackgroundJob({ jobPayload: mockJobPayload })
+      const result = await createBackgroundJob({ jobPayload: mockJobPayload })
 
       const afterTest = new Date()
 
@@ -138,13 +138,13 @@ describe('scheduleBackgroundJobFactory', () => {
           throw new Error(errorMessage)
         }
 
-        const scheduleBackgroundJob = scheduleBackgroundJobFactory({
+        const createBackgroundJob = createBackgroundJobFactory({
           storeBackgroundJob: failingStore,
           jobConfig: mockJobConfig
         })
 
         try {
-          await scheduleBackgroundJob({ jobPayload: mockJobPayload })
+          await createBackgroundJob({ jobPayload: mockJobPayload })
           expect.fail('Should have thrown an error')
         } catch (error) {
           expect(error).to.be.instanceof(Error)

packages/server/modules/fileuploads/queues/fileimports.ts

@@ -22,7 +22,7 @@ import {
 } from '@/modules/fileuploads/domain/consts'
 import type { Knex } from 'knex'
 import { migrateDbToLatest } from '@/db/migrations'
-import { scheduleBackgroundJobFactory } from '@/modules/backgroundjobs/services'
+import { createBackgroundJobFactory } from '@/modules/backgroundjobs/services/create'
 import {
   getBackgroundJobCountFactory,
   storeBackgroundJobFactory
@@ -136,7 +136,7 @@ export const initializePostgresQueue = async ({
   // migrating the DB up, the queue DB might be added based on a config
   await migrateDbToLatest({ db, region: `Queue DB for ${label}` })
 
-  const scheduleBackgroundJob = scheduleBackgroundJobFactory({
+  const createBackgroundJob = createBackgroundJobFactory({
     jobConfig: { maxAttempt: 3, timeoutMs: timeout },
     storeBackgroundJob: storeBackgroundJobFactory({
       db,
@@ -152,7 +152,7 @@ export const initializePostgresQueue = async ({
     ),
     shutdown: async () => {},
     scheduleJob: async (jobData: JobPayload) => {
-      await scheduleBackgroundJob({
+      await createBackgroundJob({
         jobPayload: { jobType: 'fileImport', payloadVersion: 1, ...jobData }
       })
     },

packages/viewer-sandbox/src/Sandbox.ts

@@ -186,7 +186,7 @@ export default class Sandbox {
       this.addStreamControls(url)
       this.addViewControls()
       this.addBatches()
-      this.properties = await this.viewer.getObjectProperties()
+      // this.properties = await this.viewer.getObjectProperties()
       this.batchesParams.totalBvhSize = this.getBVHSize()
       this.refresh()
     })

packages/viewer-sandbox/src/main.ts

@@ -135,6 +135,7 @@ const getStream = () => {
     // prettier-ignore
     // Revit sample house (good for bim-like stuff with many display meshes)
     'https://app.speckle.systems/streams/da9e320dad/commits/5388ef24b8'
+    // 'https://app.speckle.systems/streams/da9e320dad/objects/ee5d160d84090822813bc74188da34a7'
 
     //large tower
     //'https://app.speckle.systems/projects/e2a7b596f2/models/ddaf8349f5'
@@ -162,6 +163,7 @@ const getStream = () => {
     // 'https://latest.speckle.systems/streams/3ed8357f29/commits/d10f2af1ce'
     // AutoCAD NEW
     // 'https://latest.speckle.systems/streams/3ed8357f29/commits/46905429f6'
+    // 'https://latest.speckle.systems/streams/3ed8357f29/objects/95160b8d593a0ba12dd004d5fe142257'
     //Blizzard world
     // 'https://latest.speckle.systems/streams/0c6ad366c4/commits/aa1c393aec'
     //Car
@@ -355,7 +357,6 @@ const getStream = () => {
     // 'https://app.speckle.systems/streams/25d8a162af/commits/6c842a713c'
     // 'https://app.speckle.systems/streams/76e3acde68/commits/0ea3d47e6c'
     // Point cloud
-    // 'https://app.speckle.systems/streams/b920636274/commits/8df6496749'
     // 'https://multiconsult.speckle.xyz/streams/9721fe797c/objects/ff5d939a8c26bde092152d5b4a0c945d'
     // 'https://app.speckle.systems/streams/87a2be92c7/objects/803c3c413b133ee9a6631160ccb194c9'
     // 'https://latest.speckle.systems/streams/1422d91a81/commits/480d88ba68'
@@ -543,7 +544,7 @@ const getStream = () => {
     // 'https://app.speckle.systems/projects/7591c56179/models/82b94108a3'
 
     // SUPER slow tree build time (LARGE N-GONS TRIANGULATION)
-    // 'https://app.speckle.systems/projects/0edb6ef628/models/ff3d8480bc@cd83d90a2c'
+    // 'https://app.speckle.systems/projects/0edb6ef628/models/87f3fb5e2bd681d731dd048390ae3a8f'
 
     /* ObjectLoader 2 tests */
     // `https://latest.speckle.systems/projects/97750296c2/models/767b70fc63@5386a0af02`
@@ -601,6 +602,28 @@ const getStream = () => {
     // 'https://latest.speckle.systems/projects/f28ad5b38a/models/b63ebcd807'
     // Duplicate display values
     // 'https://app.speckle.systems/projects/1466fe31c6/models/2eaf0f0571'
+    // MEPS
+    // 'https://app.speckle.systems/projects/f3cee517d4/models/21f128a3ea'
+    // Tower
+    // 'https://app.speckle.systems/projects/e2a7b596f2/models/ddaf8349f5'
+
+    // Barbican
+    // 'https://app.speckle.systems/projects/32baa9291e/models/all'
+    // 'https://app.speckle.systems/streams/32baa9291e/objects/21a3621c0a3e6d2884e1315f02314313'
+    // 'https://app.speckle.systems/projects/5d723f097a/models/c05abd36b5'
+
+    //Guggenheim
+    // 'https://app.speckle.systems/projects/937d78e0a5/models/a48f0274eb'
+    // 'https://app.speckle.systems/projects/937d78e0a5/objects/0e3c61147f3a035a85a3542c7f1c7a43'
+
+    // heatherwick LARGE
+    // 'https://app.speckle.systems/projects/63a3226049/models/bdd4f553a8'
+
+    // Mirrored instances
+    // 'https://app.speckle.systems/projects/b6e95c0c63/models/024ce31c6f@a66c3956d6'
+
+    // Text with color proxy
+    // 'https://app.speckle.systems/projects/ebf93a561b/models/0a07bc3231'
   )
 }
 

packages/viewer/src/IViewer.ts

@@ -28,6 +28,13 @@ export type SpeckleObject = {
   applicationId?: string
 }
 
+export type DataChunk = {
+  id: string
+  data: number[]
+  references: number
+  processed?: boolean
+}
+
 export interface ViewerParams {
   showStats: boolean
   environmentSrc: Asset

packages/viewer/src/modules/UrlHelper.ts

@@ -299,9 +299,10 @@ async function runAllModelsQuery(
     const urls: string[] = []
     data.project.models.items.forEach(
       (element: { versions: { items: { referencedObject: string }[] } }) => {
-        urls.push(
-          `${ref.origin}/streams/${ref.projectId}/objects/${element.versions.items[0].referencedObject}`
-        )
+        if (element.versions.items.length)
+          urls.push(
+            `${ref.origin}/streams/${ref.projectId}/objects/${element.versions.items[0].referencedObject}`
+          )
       }
     )
     return urls

packages/viewer/src/modules/batching/BatchObject.ts

@@ -102,8 +102,16 @@ export class BatchObject {
       this.pivot_High
     )
   }
+  public buildAccelerationStructure(
+    position: Float32Array | Float64Array,
+    indices: Uint16Array | Uint32Array
+  ): void
+  public buildAccelerationStructure(bvh: MeshBVH): void
 
-  public buildAccelerationStructure(bvh?: MeshBVH) {
+  public buildAccelerationStructure(
+    positionOrBvh: Float32Array | Float64Array | MeshBVH,
+    indices?: Uint16Array | Uint32Array
+  ): void {
     const transform = new Matrix4().makeTranslation(
       this._localOrigin.x,
       this._localOrigin.y,
@@ -111,14 +119,15 @@ export class BatchObject {
     )
     transform.invert()
 
-    if (!bvh) {
-      const indices: number[] | undefined =
-        this._renderView.renderData.geometry.attributes?.INDEX
-      const position: number[] | undefined =
-        this._renderView.renderData.geometry.attributes?.POSITION
+    let bvh = positionOrBvh
+
+    if (!(bvh instanceof MeshBVH)) {
+      if (!indices) {
+        throw new Error(`Cannot build a BVH with only positions. Need indices too`)
+      }
       bvh = AccelerationStructure.buildBVH(
         indices,
-        position,
+        positionOrBvh as Float32Array | Float64Array,
         DefaultBVHOptions,
         transform
       )

packages/viewer/src/modules/batching/Batcher.ts

@@ -21,7 +21,7 @@ import SpeckleMesh, { TransformStorage } from '../objects/SpeckleMesh.js'
 import { SpeckleType } from '../loaders/GeometryConverter.js'
 import { type TreeNode, WorldTree } from '../../index.js'
 import { InstancedMeshBatch } from './InstancedMeshBatch.js'
-import { Geometry } from '../converter/Geometry.js'
+import { Geometry, GeometryData } from '../converter/Geometry.js'
 import { MeshBatch } from './MeshBatch.js'
 import { PointBatch } from './PointBatch.js'
 import Logger from '../utils/Logger.js'
@@ -139,23 +139,27 @@ export default class Batcher {
           needsRTE
         ) {
           rvs.forEach((nodeRv) => {
-            const geometry = nodeRv.renderData.geometry
+            const geometry = nodeRv.renderData.geometry as GeometryData
             geometry.instanced = false
-            const attribs = geometry.attributes
-            geometry.attributes = {
-              POSITION: attribs.POSITION.slice(),
-              INDEX: attribs.INDEX.slice(),
-              ...(attribs.COLOR && {
-                COLOR: attribs.COLOR.slice()
-              })
-            }
-            /**  - I don't particularly like this branch -
-             *  All instances should have a transform. But it's the easiest thing we can do
-             *  until we figure out the viewer <-> connector object duplication inconsistency
-             */
-            if (geometry.transform)
-              Geometry.transformGeometryData(geometry, geometry.transform)
-            nodeRv.computeAABB()
+            nodeRv.computeAABB(geometry.transform)
+            if ((geometry.transform?.determinant() ?? 0) < 0)
+              geometry.flipNormals = true
+            /** I don't think we need to duplicate geometry here, now that we're transforming the batch position directly */
+            // const attribs = geometry.attributes
+            // geometry.attributes = {
+            //   POSITION: attribs.POSITION.slice(),
+            //   INDEX: attribs.INDEX.slice(),
+            //   ...(attribs.COLOR && {
+            //     COLOR: attribs.COLOR.slice()
+            //   })
+            // }
+            // /**  - I don't particularly like this branch -
+            //  *  All instances should have a transform. But it's the easiest thing we can do
+            //  *  until we figure out the viewer <-> connector object duplication inconsistency
+            //  */
+            // if (geometry.transform)
+            //   Geometry.transformGeometryData(geometry, geometry.transform)
+            // nodeRv.computeAABB()
           })
           continue
         }
@@ -383,7 +387,9 @@ export default class Batcher {
     } else if (geometryType === GeometryType.POINT_CLOUD) {
       matRef = renderViews[0].renderData.renderMaterial
     } else if (geometryType === GeometryType.TEXT) {
-      matRef = renderViews[0].renderData.displayStyle
+      matRef = renderViews[0].renderData.colorMaterial
+        ? renderViews[0].renderData.colorMaterial
+        : renderViews[0].renderData.displayStyle
     }
 
     const material = this.materials.getMaterial(materialHash, matRef, geometryType)

packages/viewer/src/modules/batching/InstancedMeshBatch.ts

@@ -525,6 +525,21 @@ export class InstancedMeshBatch implements Batch {
     )
     const targetInstanceTransformBuffer = this.getCurrentTransformBuffer()
 
+    const positions =
+      this.renderViews[0].renderData.geometry.attributes?.POSITION.getFloat32Array()
+    const indicesLength =
+      this.renderViews[0].renderData.geometry.attributes?.INDEX.length ?? 0
+    const positionsLength =
+      this.renderViews[0].renderData.geometry.attributes?.POSITION.length ?? 0
+    const indices =
+      indicesLength < 65535 && positionsLength < 65535
+        ? this.renderViews[0].renderData.geometry.attributes?.INDEX.getUint16Array()
+        : this.renderViews[0].renderData.geometry.attributes?.INDEX.getUint32Array()
+    const colors =
+      this.renderViews[0].renderData.geometry.attributes?.COLOR?.getFloat32Array()
+    const normals =
+      this.renderViews[0].renderData.geometry.attributes?.NORMAL?.getFloat32Array()
+
     for (let k = 0; k < this.renderViews.length; k++) {
       /** Catering to typescript
        *  There is no unniverse where an instanced render view does not have a transform
@@ -555,13 +570,10 @@ export class InstancedMeshBatch implements Batch {
           batchObject.localOrigin.z
         )
         transform.invert()
-        const indices: number[] | undefined =
-          this.renderViews[k].renderData.geometry.attributes?.INDEX
-        const position: number[] | undefined = this.renderViews[k].renderData.geometry
-          .attributes?.POSITION as number[]
+
         instanceBVH = AccelerationStructure.buildBVH(
           indices,
-          position,
+          positions,
           DefaultBVHOptions,
           transform
         )
@@ -572,32 +584,13 @@ export class InstancedMeshBatch implements Batch {
       batchObjects.push(batchObject)
     }
 
-    const indices: number[] | undefined =
-      this.renderViews[0].renderData.geometry.attributes?.INDEX
-
-    const positions: number[] | undefined =
-      this.renderViews[0].renderData.geometry.attributes?.POSITION
-
-    const colors: number[] | undefined =
-      this.renderViews[0].renderData.geometry.attributes?.COLOR
-
-    const normals: number[] | undefined =
-      this.renderViews[0].renderData.geometry.attributes?.NORMAL
-
     /** Catering to typescript
      *  There is no unniverse where indices or positions are undefined at this point
      */
     if (!indices || !positions) {
       throw new Error(`Cannot build batch ${this.id}. Undefined indices or positions`)
     }
-    this.geometry = this.makeInstancedMeshGeometry(
-      positions.length >= 65535 || indices.length >= 65535
-        ? new Uint32Array(indices)
-        : new Uint16Array(indices),
-      new Float64Array(positions),
-      normals ? new Float32Array(normals) : undefined,
-      colors ? new Float32Array(colors) : undefined
-    )
+    this.geometry = this.makeInstancedMeshGeometry(indices, positions, normals, colors)
 
     this.mesh = new SpeckleInstancedMesh(this.geometry)
     this.mesh.setBatchObjects(batchObjects)
@@ -655,16 +648,12 @@ export class InstancedMeshBatch implements Batch {
 
   private makeInstancedMeshGeometry(
     indices: Uint32Array | Uint16Array,
-    position: Float64Array,
+    position: Float32Array,
     normal?: Float32Array,
     color?: Float32Array
   ): BufferGeometry {
     const geometry = new BufferGeometry()
     if (position) {
-      /** When RTE enabled, we'll be storing the high component of the encoding here,
-       * which considering our current encoding method is actually the original casted
-       * down float32 position!
-       */
       geometry.setAttribute('position', new Float32BufferAttribute(position, 3))
     }
 

packages/viewer/src/modules/batching/LineBatch.ts

@@ -24,6 +24,7 @@ import {
 } from './Batch.js'
 import { ObjectLayers } from '../../IViewer.js'
 import Materials from '../materials/Materials.js'
+import { ChunkArray } from '../converter/VirtualArray.js'
 
 export default class LineBatch implements Batch {
   public id: string
@@ -231,6 +232,7 @@ export default class LineBatch implements Batch {
 
   public buildBatch() {
     let attributeCount = 0
+    const rvAABB: Box3 = new Box3()
     const bounds = new Box3()
     this.renderViews.forEach((val: NodeRenderView) => {
       if (!val.renderData.geometry.attributes) {
@@ -241,14 +243,18 @@ export default class LineBatch implements Batch {
         : val.renderData.geometry.attributes.POSITION.length
       bounds.union(val.aabb)
     })
-    const position = new Float64Array(attributeCount)
+    const needsRTE = Geometry.needsRTE(bounds)
+
+    const position = needsRTE
+      ? new Float64Array(attributeCount)
+      : new Float32Array(attributeCount)
     let offset = 0
     for (let k = 0; k < this.renderViews.length; k++) {
       const geometry = this.renderViews[k].renderData.geometry
       if (!geometry.attributes) {
         throw new Error(`Cannot build batch ${this.id}. Invalid geometry`)
       }
-      let points: Array<number>
+      let points: Array<number> | ChunkArray
       /** We need to make sure the line geometry has a layout of :
        *  start(x,y,z), end(x,y,z), start(x,y,z), end(x,y,z)... etc
        *  Some geometries have that inherent form, some don't
@@ -258,19 +264,30 @@ export default class LineBatch implements Batch {
         points = new Array(2 * length)
 
         for (let i = 0; i < length; i += 3) {
-          points[2 * i] = geometry.attributes.POSITION[i]
-          points[2 * i + 1] = geometry.attributes.POSITION[i + 1]
-          points[2 * i + 2] = geometry.attributes.POSITION[i + 2]
+          points[2 * i] = geometry.attributes.POSITION.get(i)
+          points[2 * i + 1] = geometry.attributes.POSITION.get(i + 1)
+          points[2 * i + 2] = geometry.attributes.POSITION.get(i + 2)
 
-          points[2 * i + 3] = geometry.attributes.POSITION[i + 3]
-          points[2 * i + 4] = geometry.attributes.POSITION[i + 4]
-          points[2 * i + 5] = geometry.attributes.POSITION[i + 5]
+          points[2 * i + 3] = geometry.attributes.POSITION.get(i + 3)
+          points[2 * i + 4] = geometry.attributes.POSITION.get(i + 4)
+          points[2 * i + 5] = geometry.attributes.POSITION.get(i + 5)
         }
+        position.set(points, offset)
       } else {
         points = geometry.attributes.POSITION
+        geometry.attributes.POSITION.copyToBuffer(position, offset)
       }
 
-      position.set(points, offset)
+      const positionSubArray = position.subarray(offset, offset + points.length)
+      Geometry.transformArray(positionSubArray, geometry.transform, 0, points.length)
+      /** We re-compute the render view aabb based on transformed geometry
+       *  We do this because some transforms like non-uniform scaling can produce incorrect results
+       *  if we compute an aabb from original geometry then apply the transform. That's why we compute
+       *  an aabb from the transformed geometry here and set it in the rv
+       */
+      rvAABB.setFromArray(positionSubArray)
+      this.renderViews[k].aabb = rvAABB
+
       this.renderViews[k].setBatchData(this.id, offset / 6, points.length / 6)
 
       offset += points.length
@@ -319,10 +336,15 @@ export default class LineBatch implements Batch {
     return material
   }
 
-  private makeLineGeometry(position: Float64Array): LineSegmentsGeometry {
+  private makeLineGeometry(
+    position: Float64Array | Float32Array
+  ): LineSegmentsGeometry {
     const geometry = new LineSegmentsGeometry()
     /** This will set the instanceStart and instanceEnd attributes. These will be our high parts */
-    geometry.setPositions(new Float32Array(position))
+    if (position instanceof Float64Array)
+      /** We need to re-allocate because there is no way to cast it down to float32. If we pass in a Float64Array, three.js will do it anyway */
+      geometry.setPositions(new Float32Array(position))
+    else geometry.setPositions(position)
 
     const buffer = new Float32Array(position.length + position.length / 3)
     this.colorBuffer = new InstancedInterleavedBuffer(buffer, 8, 1) // rgba, rgba

packages/viewer/src/modules/batching/MeshBatch.ts

@@ -193,6 +193,7 @@ export class MeshBatch extends PrimitiveBatch {
   public buildBatch(): Promise<void> {
     let indicesCount = 0
     let attributeCount = 0
+    const rvAABB: Box3 = new Box3()
     const bounds: Box3 = new Box3()
     for (let k = 0; k < this.renderViews.length; k++) {
       const ervee = this.renderViews[k]
@@ -209,11 +210,17 @@ export class MeshBatch extends PrimitiveBatch {
       attributeCount += ervee.renderData.geometry.attributes.POSITION.length
       bounds.union(ervee.aabb)
     }
+    const needsRTE = Geometry.needsRTE(bounds)
 
     const hasVertexColors =
       this.renderViews[0].renderData.geometry.attributes?.COLOR !== undefined
-    const indices = new Uint32Array(indicesCount)
-    const position = new Float64Array(attributeCount)
+    const indices =
+      attributeCount >= 65535 || indicesCount >= 65535
+        ? new Uint32Array(indicesCount)
+        : new Uint16Array(indicesCount)
+    const position = needsRTE
+      ? new Float64Array(attributeCount)
+      : new Float32Array(attributeCount)
     const color = new Float32Array(hasVertexColors ? attributeCount : 0)
     color.fill(1)
     const batchIndices = new Float32Array(attributeCount / 3)
@@ -228,49 +235,90 @@ export class MeshBatch extends PrimitiveBatch {
       /** Catering to typescript
        *  There is no unniverse where indices or positions are undefined at this point
        */
-      if (!geometry.attributes || !geometry.attributes.INDEX) {
+      if (!geometry.attributes || !geometry.attributes?.INDEX) {
         throw new Error(`Cannot build batch ${this.id}. Invalid geometry, or indices`)
       }
-      indices.set(
-        geometry.attributes.INDEX.map((val) => val + offset / 3),
-        arrayOffset
+
+      geometry.attributes?.INDEX.copyToBuffer(indices, arrayOffset)
+      const indicesSubArray = indices.subarray(
+        arrayOffset,
+        arrayOffset + geometry.attributes?.INDEX.length
       )
-      position.set(geometry.attributes.POSITION, offset)
-      if (geometry.attributes.COLOR) color.set(geometry.attributes.COLOR, offset)
+
+      geometry.attributes?.POSITION.copyToBuffer(position, offset)
+      const positionSubarray = position.subarray(
+        offset,
+        offset +
+          (this.renderViews[k].renderData.geometry.attributes?.POSITION.length ?? 0)
+      )
+      /** We transform the copied geometry so that we do not alter original chunk data which might be shared */
+      Geometry.transformArray(
+        positionSubarray,
+        geometry.transform,
+        0,
+        geometry.attributes?.POSITION.length
+      )
+
+      if (geometry.attributes.COLOR) {
+        geometry.attributes?.COLOR.copyToBuffer(color, offset)
+      }
 
       /** We either copy over the provided vertex normals */
       if (geometry.attributes.NORMAL) {
-        normals.set(geometry.attributes.NORMAL, offset)
+        geometry.attributes?.NORMAL.copyToBuffer(normals, offset)
+        if (geometry.flipNormals) {
+          Geometry.flipNormalsBuffer(
+            normals.subarray(offset, offset + geometry.attributes?.NORMAL.length)
+          )
+        }
       } else {
         /** Either we compute them ourselves */
-        Geometry.computeVertexNormalsBuffer(
+        Geometry.computeVertexNormalsBufferVirtual(
           normals.subarray(
             offset,
-            offset + geometry.attributes.POSITION.length
+            offset +
+              (this.renderViews[k].renderData.geometry.attributes?.POSITION.length ?? 0)
           ) as unknown as number[],
           geometry.attributes.POSITION,
-          geometry.attributes.INDEX
+          geometry.attributes.INDEX,
+          geometry.flipNormals
         )
       }
-      batchIndices.fill(
-        k,
-        offset / 3,
-        offset / 3 + geometry.attributes.POSITION.length / 3
-      )
+      batchIndices.fill(k, offset / 3, offset / 3 + geometry.attributes.POSITION.length)
       this.renderViews[k].setBatchData(
         this.id,
         arrayOffset,
         geometry.attributes.INDEX.length,
         offset / 3,
-        offset / 3 + geometry.attributes.POSITION.length / 3
+        offset / 3 + geometry.attributes.POSITION.length
       )
 
+      /** We re-compute the render view aabb based on transformed geometry
+       *  We do this because some transforms like non-uniform scaling can produce incorrect results
+       *  if we compute an aabb from original geometry then apply the transform. That's why we compute
+       *  an aabb from the transformed geometry here and set it in the rv
+       */
+      rvAABB.setFromArray(positionSubarray)
+      this.renderViews[k].aabb = rvAABB
+
       const batchObject = new BatchObject(this.renderViews[k], k)
-      batchObject.buildAccelerationStructure()
+      batchObject.buildAccelerationStructure(positionSubarray, indicesSubArray)
       batchObjects.push(batchObject)
 
+      indices.set(
+        batchObject.accelerationStructure.bvh.geometry.index?.array as number[],
+        arrayOffset
+      )
+
+      /** Re-index the indices inside the batch */
+      for (let i = 0; i < indicesSubArray.length; i++) {
+        indicesSubArray[i] = indicesSubArray[i] + offset / 3
+      }
+
       offset += geometry.attributes.POSITION.length
       arrayOffset += geometry.attributes.INDEX.length
+
+      this.renderViews[k].disposeGeometry()
     }
 
     const geometry = this.makeMeshGeometry(
@@ -280,7 +328,7 @@ export class MeshBatch extends PrimitiveBatch {
       batchIndices,
       hasVertexColors ? color : undefined
     )
-    const needsRTE = Geometry.needsRTE(bounds)
+
     if (needsRTE) Geometry.updateRTEGeometry(geometry, position)
 
     this.primitive = new SpeckleMesh(geometry, needsRTE)
@@ -296,16 +344,16 @@ export class MeshBatch extends PrimitiveBatch {
     this.primitive.frustumCulled = false
     this.primitive.geometry.addGroup(0, this.getCount(), 0)
 
-    batchObjects.forEach((element: BatchObject) => {
-      element.renderView.disposeGeometry()
-    })
+    // batchObjects.forEach((element: BatchObject) => {
+    //   element.renderView.disposeGeometry()
+    // })
 
     return Promise.resolve()
   }
 
   protected makeMeshGeometry(
     indices: Uint32Array | Uint16Array,
-    position: Float64Array,
+    position: Float64Array | Float32Array,
     normals: Float32Array,
     batchIndices: Float32Array,
     color?: Float32Array
@@ -313,10 +361,10 @@ export class MeshBatch extends PrimitiveBatch {
     const geometry = new BufferGeometry()
     if (position.length >= 65535 || indices.length >= 65535) {
       this.indexBuffer0 = new Uint32BufferAttribute(indices, 1)
-      this.indexBuffer1 = new Uint32BufferAttribute(new Uint32Array(indices.length), 1)
+      this.indexBuffer1 = new Uint32BufferAttribute(indices, 1)
     } else {
       this.indexBuffer0 = new Uint16BufferAttribute(indices, 1)
-      this.indexBuffer1 = new Uint16BufferAttribute(new Uint16Array(indices.length), 1)
+      this.indexBuffer1 = new Uint16BufferAttribute(indices, 1)
     }
     geometry.setIndex(this.indexBuffer0)
 

packages/viewer/src/modules/batching/PointBatch.ts

@@ -158,6 +158,7 @@ export class PointBatch extends PrimitiveBatch {
 
   public buildBatch(): Promise<void> {
     let attributeCount = 0
+    const rvAABB: Box3 = new Box3()
     const bounds = new Box3()
     for (let k = 0; k < this.renderViews.length; k++) {
       const ervee = this.renderViews[k]
@@ -170,9 +171,17 @@ export class PointBatch extends PrimitiveBatch {
       attributeCount += ervee.renderData.geometry.attributes.POSITION.length
       bounds.union(ervee.aabb)
     }
-    const position = new Float64Array(attributeCount)
-    const color = new Float32Array(attributeCount).fill(1)
-    const index = new Int32Array(attributeCount / 3)
+    const needsRTE = Geometry.needsRTE(bounds)
+    const hasVertexColors =
+      this.renderViews[0].renderData.geometry.attributes?.COLOR !== undefined
+    const needsInt32Indices = attributeCount >= 65535
+    const position = needsRTE
+      ? new Float64Array(attributeCount)
+      : new Float32Array(attributeCount)
+    const color = new Float32Array(hasVertexColors ? attributeCount : 0).fill(1)
+    const index = needsInt32Indices
+      ? new Uint32Array(attributeCount / 3)
+      : new Uint16Array(attributeCount / 3)
     let offset = 0
     let indexOffset = 0
     for (let k = 0; k < this.renderViews.length; k++) {
@@ -180,14 +189,39 @@ export class PointBatch extends PrimitiveBatch {
       if (!geometry.attributes) {
         throw new Error(`Cannot build batch ${this.id}. Invalid geometry, or indices`)
       }
-      position.set(geometry.attributes.POSITION, offset)
-      if (geometry.attributes.COLOR) color.set(geometry.attributes.COLOR, offset)
+
+      geometry.attributes?.POSITION.copyToBuffer(position, offset)
+      const positionSubarray = position.subarray(
+        offset,
+        offset +
+          (this.renderViews[k].renderData.geometry.attributes?.POSITION.length ?? 0)
+      )
+
+      Geometry.transformArray(
+        positionSubarray,
+        geometry.transform,
+        0,
+        geometry.attributes?.POSITION.length
+      )
+      if (geometry.attributes.COLOR) {
+        geometry.attributes?.COLOR.copyToBuffer(color, offset)
+      }
       index.set(
-        new Int32Array(geometry.attributes.POSITION.length / 3).map(
-          (_value, index) => index + indexOffset
-        ),
+        (needsInt32Indices
+          ? new Uint32Array(geometry.attributes.POSITION.length / 3)
+          : new Uint16Array(geometry.attributes.POSITION.length / 3)
+        ).map((_value, index) => index + indexOffset),
         indexOffset
       )
+
+      /** We re-compute the render view aabb based on transformed geometry
+       *  We do this because some transforms like non-uniform scaling can produce incorrect results
+       *  if we compute an aabb from original geometry then apply the transform. That's why we compute
+       *  an aabb from the transformed geometry here and set it in the rv
+       */
+      rvAABB.setFromArray(positionSubarray)
+      this.renderViews[k].aabb = rvAABB
+
       this.renderViews[k].setBatchData(
         this.id,
         offset / 3,
@@ -201,7 +235,7 @@ export class PointBatch extends PrimitiveBatch {
     }
     const geometry = this.makePointGeometry(index, position, color)
 
-    if (Geometry.needsRTE(bounds)) {
+    if (needsRTE) {
       Geometry.updateRTEGeometry(geometry, position)
       if (!this.batchMaterial.defines) this.batchMaterial.defines = {}
       this.batchMaterial.defines['USE_RTE'] = ' '
@@ -221,8 +255,8 @@ export class PointBatch extends PrimitiveBatch {
   }
 
   protected makePointGeometry(
-    index: Int32Array,
-    position: Float64Array,
+    index: Uint32Array | Uint16Array,
+    position: Float64Array | Float32Array,
     color: Float32Array
   ): BufferGeometry {
     const geometry = new BufferGeometry()

packages/viewer/src/modules/batching/TextBatch.ts

@@ -144,8 +144,8 @@ export default class TextBatch implements Batch {
    * - Even if, the **text batch does not use the materials in it's draw groups**, it emulates the behavior as if it would
    */
   public setBatchBuffers(ranges: BatchUpdateRange[]): void {
-    console.warn(' Groups -> ', this.mesh.groups)
-    console.warn(' Ranges -> ', ranges)
+    // console.warn(' Groups -> ', this.mesh.groups)
+    // console.warn(' Ranges -> ', ranges)
     const splitRanges: BatchUpdateRange[] = []
     ranges.forEach((range: BatchUpdateRange) => {
       for (let k = 0; k < range.count; k++) {
@@ -321,7 +321,7 @@ export default class TextBatch implements Batch {
           screenOriented: boolean
         }
         const text = new Text()
-        this.renderViews[k].renderData.geometry.bakeTransform?.decompose(
+        this.renderViews[k].renderData.geometry.transform?.decompose(
           text.position,
           text.quaternion,
           text.scale
@@ -349,24 +349,23 @@ export default class TextBatch implements Batch {
           /** We're using visibleBounds for a better fit */
           const bounds = textRenderInfo.visibleBounds
           // console.log('bounds -> ', bounds)
-          const vertices = []
-          vertices.push(
-            bounds[0],
-            bounds[3],
-            0,
-            bounds[2],
-            bounds[3],
-            0,
-            bounds[0],
-            bounds[1],
-            0,
-            bounds[2],
-            bounds[1],
-            0
-          )
+          const vertices = new Float32Array(12)
+          vertices[0] = bounds[0]
+          vertices[1] = bounds[3]
+          vertices[2] = 0
+          vertices[3] = bounds[2]
+          vertices[4] = bounds[3]
+          vertices[5] = 0
+          vertices[6] = bounds[0]
+          vertices[7] = bounds[1]
+          vertices[8] = 0
+          vertices[9] = bounds[2]
+          vertices[10] = bounds[1]
+          vertices[11] = 0
+
           box.setFromArray(vertices)
           box.applyMatrix4(
-            this.renderViews[k].renderData.geometry.bakeTransform || new Matrix4()
+            this.renderViews[k].renderData.geometry.transform || new Matrix4()
           )
 
           needsRTE ||= Geometry.needsRTE(box)
@@ -374,7 +373,7 @@ export default class TextBatch implements Batch {
           const geometry = text.geometry
           geometry.computeBoundingBox()
           const textBvh = AccelerationStructure.buildBVH(
-            geometry.index?.array as number[],
+            geometry.index?.array as unknown as Uint16Array | Uint32Array,
             vertices,
             DefaultBVHOptions
           )

packages/viewer/src/modules/batching/TextBatchObject.ts

@@ -7,8 +7,8 @@ export class TextBatchObject extends BatchObject {
 
   public constructor(renderView: NodeRenderView, batchIndex: number) {
     super(renderView, batchIndex)
-    if (renderView.renderData.geometry.bakeTransform)
-      this.textTransform.copy(renderView.renderData.geometry.bakeTransform)
+    if (renderView.renderData.geometry.transform)
+      this.textTransform.copy(renderView.renderData.geometry.transform)
     /** TO DO: Not sure we should do this */
     this.transform.copy(this.textTransform)
     this.transformInv.copy(new Matrix4().copy(this.textTransform).invert())

packages/viewer/src/modules/converter/Geometry.ts

@@ -6,18 +6,19 @@ import {
   Float32BufferAttribute,
   InstancedInterleavedBuffer,
   InterleavedBufferAttribute,
+  MathUtils,
   Matrix4,
   Vector2,
   Vector3,
   Vector4
 } from 'three'
-import { type SpeckleObject } from '../../IViewer.js'
+import { DataChunk, type SpeckleObject } from '../../IViewer.js'
 import { getRelativeOffset, makePerspectiveProjection } from '../Helpers.js'
 import earcut from 'earcut'
+import { ChunkArray } from './VirtualArray.js'
 
 const vecBuff0: Vector3 = new Vector3()
 const floatArrayBuff: Float32Array = new Float32Array(16)
-Vector3
 
 export enum GeometryAttributes {
   POSITION = 'POSITION',
@@ -28,18 +29,28 @@ export enum GeometryAttributes {
   INDEX = 'INDEX'
 }
 
+type AttributeValue = ChunkArray
+
+// Required keys
+type RequiredKeys = GeometryAttributes.POSITION | GeometryAttributes.INDEX
+
+// Optional keys
+type OptionalKeys = Exclude<GeometryAttributes, RequiredKeys>
+
+// Final shape: required + optional keys
+type GeometryAttributesShape = {
+  [K in RequiredKeys]: AttributeValue
+} & {
+  [K in OptionalKeys]?: AttributeValue
+}
+
 export interface GeometryData {
-  attributes:
-    | ({
-        [GeometryAttributes.POSITION]: number[]
-      } & Partial<
-        Record<Exclude<GeometryAttributes, GeometryAttributes.POSITION>, number[]>
-      >)
-    | null
+  attributes: GeometryAttributesShape | null
   bakeTransform: Matrix4 | null
   transform: Matrix4 | null
   metaData?: SpeckleObject
   instanced?: boolean
+  flipNormals?: boolean
 }
 
 export class Geometry {
@@ -70,11 +81,7 @@ export class Geometry {
 
       Geometry.DoubleToHighLowBuffer(doublePositions, position_low, position_high)
 
-      const instanceBufferLow = new InstancedInterleavedBuffer(
-        new Float32Array(position_low),
-        6,
-        1
-      ) // xyz, xyz
+      const instanceBufferLow = new InstancedInterleavedBuffer(position_low, 6, 1) // xyz, xyz
       geometry.setAttribute(
         'instanceStartLow',
         new InterleavedBufferAttribute(instanceBufferLow, 3, 0)
@@ -87,7 +94,7 @@ export class Geometry {
   }
 
   static mergeGeometryAttribute(
-    attributes: (number[] | undefined)[],
+    attributes: AttributeValue[],
     target: Float32Array | Float64Array
   ): ArrayLike<number> {
     let offset = 0
@@ -96,15 +103,15 @@ export class Geometry {
       if (!attribute || !target) {
         throw new Error('Cannot merge geometries. Indices or positions are undefined')
       }
-      target.set(attribute, offset)
+      attribute.copyToBuffer(target, offset)
       offset += attribute.length
     }
     return target
   }
 
   static mergeIndexAttribute(
-    indexAttributes: (number[] | undefined)[],
-    positionAttributes: (number[] | undefined)[]
+    indexAttributes: AttributeValue[],
+    positionAttributes: AttributeValue[]
   ): number[] {
     let indexOffset = 0
     const mergedIndex = []
@@ -117,7 +124,7 @@ export class Geometry {
       }
 
       for (let j = 0; j < index.length; ++j) {
-        mergedIndex.push(index[j] + indexOffset / 3)
+        mergedIndex.push(index.get(j) + indexOffset / 3)
       }
 
       indexOffset += positions.length
@@ -139,53 +146,59 @@ export class Geometry {
         Geometry.transformGeometryData(geometries[i], geometries[i].bakeTransform)
     }
 
-    if (sampleAttributes && sampleAttributes[GeometryAttributes.INDEX]) {
-      const indexAttributes: (number[] | undefined)[] = geometries.map(
-        (item: GeometryData) => {
-          /** Catering to typescript */
-          if (!item.attributes) return
-          return item.attributes[GeometryAttributes.INDEX]
-        }
-      )
-      const positionAttributes: (number[] | undefined)[] = geometries.map((item) => {
+    if (sampleAttributes && sampleAttributes.INDEX) {
+      const indexAttributes = geometries.map((item: GeometryData) => {
         /** Catering to typescript */
         if (!item.attributes) return
-        return item.attributes[GeometryAttributes.POSITION]
-      })
+        return item.attributes.INDEX
+      }) as ChunkArray[]
+      const positionAttributes = geometries.map((item) => {
+        /** Catering to typescript */
+        if (!item.attributes) return
+        return item.attributes.POSITION
+      }) as ChunkArray[]
       /** o_0 Catering to typescript*/
       if (mergedGeometry.attributes)
-        mergedGeometry.attributes[GeometryAttributes.INDEX] =
-          Geometry.mergeIndexAttribute(indexAttributes, positionAttributes)
+        mergedGeometry.attributes.INDEX = new ChunkArray([
+          {
+            data: Geometry.mergeIndexAttribute(indexAttributes, positionAttributes),
+            id: MathUtils.generateUUID(),
+            references: 1
+          }
+        ])
     }
 
     for (const k in sampleAttributes) {
       if (k !== GeometryAttributes.INDEX) {
-        const attributes: (number[] | undefined)[] = geometries.map((item) => {
+        const attributes: ChunkArray[] = geometries.map((item) => {
           /** Catering to typescript */
           if (!item.attributes) return
-          return item.attributes[k as GeometryAttributes] as number[]
-        })
+          return item.attributes[k as GeometryAttributes]
+        }) as ChunkArray[]
         /** Catering to typescript */
-        if (mergedGeometry.attributes)
-          mergedGeometry.attributes[k as GeometryAttributes] =
-            Geometry.mergeGeometryAttribute(
-              attributes,
-              k === GeometryAttributes.POSITION
-                ? new Float64Array(
-                    attributes.reduce((prev, cur) => {
-                      /** Catering to typescript */
-                      if (!cur) return 0
-                      return prev + cur.length
-                    }, 0)
-                  )
-                : new Float32Array(
-                    attributes.reduce((prev, cur) => {
-                      /** Catering to typescript */
-                      if (!cur) return 0
-                      return prev + cur.length
-                    }, 0)
-                  )
-            ) as number[]
+        if (mergedGeometry.attributes) {
+          const mergedData = Geometry.mergeGeometryAttribute(
+            attributes,
+            k === GeometryAttributes.POSITION
+              ? new Float64Array(
+                  attributes.reduce((prev, cur) => {
+                    /** Catering to typescript */
+                    if (!cur) return 0
+                    return prev + cur.length
+                  }, 0)
+                )
+              : new Float32Array(
+                  attributes.reduce((prev, cur) => {
+                    /** Catering to typescript */
+                    if (!cur) return 0
+                    return prev + cur.length
+                  }, 0)
+                )
+          ) as number[]
+          mergedGeometry.attributes[k as GeometryAttributes] = new ChunkArray([
+            { data: mergedData, id: MathUtils.generateUUID(), references: 1 }
+          ])
+        }
       }
     }
 
@@ -202,23 +215,76 @@ export class Geometry {
     if (!geometryData.attributes) return
     if (!geometryData.attributes.POSITION) return
     if (!m) return
+    if (Geometry.isMatrix4Identity(m)) return
 
     const e = m.elements
+    geometryData.attributes.POSITION.chunkArray.forEach((chunk: DataChunk) => {
+      for (let k = 0; k < chunk.data.length; k += 3) {
+        const x = chunk.data[k],
+          y = chunk.data[k + 1],
+          z = chunk.data[k + 2]
+        const w = 1 / (e[3] * x + e[7] * y + e[11] * z + e[15])
 
-    for (let k = 0; k < geometryData.attributes.POSITION.length; k += 3) {
-      const x = geometryData.attributes.POSITION[k],
-        y = geometryData.attributes.POSITION[k + 1],
-        z = geometryData.attributes.POSITION[k + 2]
+        chunk.data[k] = (e[0] * x + e[4] * y + e[8] * z + e[12]) * w
+        chunk.data[k + 1] = (e[1] * x + e[5] * y + e[9] * z + e[13]) * w
+        chunk.data[k + 2] = (e[2] * x + e[6] * y + e[10] * z + e[14]) * w
+      }
+    })
+  }
+
+  public static transformArray(
+    array: number[] | Float32Array | Float64Array,
+    m: Matrix4 | null,
+    offset?: number,
+    count?: number
+  ) {
+    if (!m) return
+    if (Geometry.isMatrix4Identity(m)) return
+
+    const e = m.elements
+    offset = offset || 0
+    count = count || array.length
+    for (let k = offset; k < offset + count; k += 3) {
+      const x = array[k],
+        y = array[k + 1],
+        z = array[k + 2]
       const w = 1 / (e[3] * x + e[7] * y + e[11] * z + e[15])
 
-      geometryData.attributes.POSITION[k] = (e[0] * x + e[4] * y + e[8] * z + e[12]) * w
-      geometryData.attributes.POSITION[k + 1] =
-        (e[1] * x + e[5] * y + e[9] * z + e[13]) * w
-      geometryData.attributes.POSITION[k + 2] =
-        (e[2] * x + e[6] * y + e[10] * z + e[14]) * w
+      array[k] = (e[0] * x + e[4] * y + e[8] * z + e[12]) * w
+      array[k + 1] = (e[1] * x + e[5] * y + e[9] * z + e[13]) * w
+      array[k + 2] = (e[2] * x + e[6] * y + e[10] * z + e[14]) * w
     }
   }
 
+  public static isMatrix4Identity(matrix: Matrix4) {
+    const e = matrix.elements
+
+    // Check all off-diagonal elements first
+    if (
+      e[1] !== 0 ||
+      e[2] !== 0 ||
+      e[3] !== 0 ||
+      e[4] !== 0 ||
+      e[6] !== 0 ||
+      e[7] !== 0 ||
+      e[8] !== 0 ||
+      e[9] !== 0 ||
+      e[11] !== 0 ||
+      e[12] !== 0 ||
+      e[13] !== 0 ||
+      e[14] !== 0
+    ) {
+      return false
+    }
+
+    // Now check diagonals
+    if (e[0] !== 1 || e[5] !== 1 || e[10] !== 1 || e[15] !== 1) {
+      return false
+    }
+
+    return true
+  }
+
   public static unpackColors(int32Colors: number[]): number[] {
     const colors = new Array<number>(int32Colors.length * 3)
     for (let i = 0; i < int32Colors.length; i++) {
@@ -431,9 +497,82 @@ export class Geometry {
     }
   }
 
+  public static computeVertexNormalsBufferVirtual(
+    buffer: number[],
+    position: ChunkArray,
+    index: ChunkArray,
+    flip: boolean = false
+  ) {
+    const pA = new Vector3(),
+      pB = new Vector3(),
+      pC = new Vector3()
+    const nA = new Vector3(),
+      nB = new Vector3(),
+      nC = new Vector3()
+    const cb = new Vector3(),
+      ab = new Vector3()
+
+    // indexed elements
+    for (let i = 0, il = index.length; i < il; i += 3) {
+      const vA = index.get(i + 0)
+      const vB = index.get(i + 1)
+      const vC = index.get(i + 2)
+      pA.set(position.get(vA * 3), position.get(vA * 3 + 1), position.get(vA * 3 + 2))
+      pB.set(position.get(vB * 3), position.get(vB * 3 + 1), position.get(vB * 3 + 2))
+      pC.set(position.get(vC * 3), position.get(vC * 3 + 1), position.get(vC * 3 + 2))
+
+      cb.subVectors(pC, pB)
+      ab.subVectors(pA, pB)
+      cb.cross(ab)
+
+      nA.fromArray(buffer, vA * 3)
+      nB.fromArray(buffer, vB * 3)
+      nC.fromArray(buffer, vC * 3)
+
+      nA.add(cb)
+      nB.add(cb)
+      nC.add(cb)
+
+      if (flip) {
+        nA.normalize()
+        nB.normalize()
+        nC.normalize()
+
+        nA.negate()
+        nB.negate()
+        nC.negate()
+      }
+
+      buffer[vA * 3] = nA.x
+      buffer[vA * 3 + 1] = nA.y
+      buffer[vA * 3 + 2] = nA.z
+
+      buffer[vB * 3] = nB.x
+      buffer[vB * 3 + 1] = nB.y
+      buffer[vB * 3 + 2] = nB.z
+
+      buffer[vC * 3] = nC.x
+      buffer[vC * 3 + 1] = nC.y
+      buffer[vC * 3 + 2] = nC.z
+    }
+  }
+
+  // ¯\_(ツ)_/¯
+  public static flipNormalsBuffer(buffer: Float32Array) {
+    const vec3 = new Vector3()
+    for (let k = 0; k < buffer.length; k += 3) {
+      vec3.set(buffer[k], buffer[k + 1], buffer[k + 2])
+      vec3.normalize()
+      vec3.negate()
+      buffer[k] = vec3.x
+      buffer[k + 1] = vec3.y
+      buffer[k + 2] = vec3.z
+    }
+  }
+
   public static computeVertexNormals(
     buffer: BufferGeometry,
-    doublePositions: Float64Array
+    positions: Float64Array | Float32Array
   ) {
     const index = buffer.index
     const positionAttribute = buffer.getAttribute('position')
@@ -470,9 +609,9 @@ export class Geometry {
           const vB = index.getX(i + 1)
           const vC = index.getX(i + 2)
 
-          pA.fromArray(doublePositions, vA * 3)
-          pB.fromArray(doublePositions, vB * 3)
-          pC.fromArray(doublePositions, vC * 3)
+          pA.fromArray(positions, vA * 3)
+          pB.fromArray(positions, vB * 3)
+          pC.fromArray(positions, vC * 3)
 
           cb.subVectors(pC, pB)
           ab.subVectors(pA, pB)
@@ -495,9 +634,9 @@ export class Geometry {
 
         for (let i = 0, il = positionAttribute.count; i < il; i += 3) {
           /** This is done blind. Don't think speckle supports non-indexed geometry */
-          pA.fromArray(doublePositions, i * 3)
-          pB.fromArray(doublePositions, i * 3 + 1)
-          pC.fromArray(doublePositions, i * 3 + 2)
+          pA.fromArray(positions, i * 3)
+          pB.fromArray(positions, i * 3 + 1)
+          pC.fromArray(positions, i * 3 + 2)
 
           cb.subVectors(pC, pB)
           ab.subVectors(pA, pB)

packages/viewer/src/modules/converter/MeshTriangulationHelper.js

@@ -1,7 +1,17 @@
+/* eslint-disable camelcase */
+import { Triangle, Vector3 } from 'three'
+
 /**
  * Set of functions to triangulate n-gon faces (i.e. polygon faces with an arbitrary (n) number of vertices).
  * This class is a JavaScript port of https://github.com/specklesystems/speckle-sharp/blob/main/Objects/Objects/Utils/MeshTriangulationHelper.cs
  */
+const _vec30 = new Vector3()
+const _vec31 = new Vector3()
+const _vec32 = new Vector3()
+const _vec33 = new Vector3()
+const _normal = new Vector3()
+const _triangle = new Triangle()
+
 export default class MeshTriangulationHelper {
   /**
    * Calculates the triangulation of the face at given faceIndex.
@@ -9,30 +19,39 @@ export default class MeshTriangulationHelper {
    * @param {Number}   faceIndex      The index of the face's cardinality indicator `n`
    * @param {Number[]}   faces      The list of faces in the mesh
    * @param {Number[]}   vertices   The list of vertices in the mesh
-   * @return {Number[]} flat list of triangle faces (without cardinality indicators)
+   * @return {Number} flat list of triangle faces (without cardinality indicators)
    */
-  static triangulateFace(faceIndex, faces, vertices) {
-    let n = faces[faceIndex]
+  static triangulateFace(
+    faceIndex,
+    faces,
+    vertices,
+    /** Purists rolling over in their graves because of this */
+    _inout_targetArray,
+    _in_offset
+  ) {
+    let n = faces.get(faceIndex)
     if (n < 3) n += 3 // 0 -> 3, 1 -> 4
 
     //Converts from relative to absolute index (returns index in mesh.vertices list)
+    /** Why doesn't javascript have a means to inline functions?! */
     function asIndex(v) {
       return faceIndex + v + 1
     }
 
     //Gets vertex from a relative vert index
-    function V(v) {
-      const index = faces[asIndex(v)] * 3
-      return new Vector3(vertices[index], vertices[index + 1], vertices[index + 2])
+    function V(v, target) {
+      const index = faces.get(asIndex(v)) * 3
+      target.x = vertices.get(index)
+      target.y = vertices.get(index + 1)
+      target.z = vertices.get(index + 2)
+      return target
     }
 
-    const triangleFaces = Array((n - 2) * 3)
-
     //Calculate face normal using the Newell Method
-    const faceNormal = new Vector3(0, 0, 0)
+    const faceNormal = _normal
     for (let ii = n - 1, jj = 0; jj < n; ii = jj, jj++) {
-      const iPos = V(ii)
-      const jPos = V(jj)
+      const iPos = V(ii, _vec30)
+      const jPos = V(jj, _vec31)
       faceNormal.x += (jPos.y - iPos.y) * (iPos.z + jPos.z) // projection on yz
       faceNormal.y += (jPos.z - iPos.z) * (iPos.x + jPos.x) // projection on xz
       faceNormal.z += (jPos.x - iPos.x) * (iPos.y + jPos.y) // projection on xy
@@ -40,8 +59,8 @@ export default class MeshTriangulationHelper {
     faceNormal.normalize()
 
     //Set up previous and next links to effectively form a double-linked vertex list
-    const prev = Array(n)
-    const next = Array(n)
+    const prev = [] //new Array(n)
+    const next = [] //new Array(n)
     for (let j = 0; j < n; j++) {
       prev[j] = j - 1
       next[j] = j + 1
@@ -52,20 +71,25 @@ export default class MeshTriangulationHelper {
     //Start clipping ears until we are left with a triangle
     let i = 0
     let counter = 0
+    let localOffset = 0
     while (n >= 3) {
       let isEar = true
 
       //If we are the last triangle or we have exhausted our vertices, the below statement will be false
       if (n > 3 && counter < n) {
-        const prevVertex = V(prev[i])
-        const earVertex = V(i)
-        const nextVertex = V(next[i])
+        const prevVertex = V(prev[i], _vec30)
+        const earVertex = V(i, _vec31)
+        const nextVertex = V(next[i], _vec32)
 
-        if (this.triangleIsCCW(faceNormal, prevVertex, earVertex, nextVertex)) {
+        _triangle.a.copy(prevVertex)
+        _triangle.b.copy(earVertex)
+        _triangle.c.copy(nextVertex)
+
+        if (_triangle.isFrontFacing(faceNormal)) {
           let k = next[next[i]]
 
           do {
-            if (this.testPointTriangle(V(k), prevVertex, earVertex, nextVertex)) {
+            if (_triangle.containsPoint(V(k, _vec33))) {
               isEar = false
               break
             }
@@ -78,10 +102,13 @@ export default class MeshTriangulationHelper {
       }
 
       if (isEar) {
-        const a = faces[asIndex(i)]
-        const b = faces[asIndex(next[i])]
-        const c = faces[asIndex(prev[i])]
-        triangleFaces.push(a, b, c)
+        const a = faces.get(asIndex(i))
+        const b = faces.get(asIndex(next[i]))
+        const c = faces.get(asIndex(prev[i]))
+        _inout_targetArray[_in_offset + localOffset] = a
+        _inout_targetArray[_in_offset + localOffset + 1] = b
+        _inout_targetArray[_in_offset + localOffset + 2] = c
+        localOffset += 3
 
         next[prev[i]] = next[i]
         prev[next[i]] = prev[i]
@@ -94,89 +121,6 @@ export default class MeshTriangulationHelper {
       }
     }
 
-    return triangleFaces
-  }
-
-  /**
-   * Tests if point v is within the triangle *abc*.
-   * @param {Vector3} v
-   * @param {Vector3} a
-   * @param {Vector3} b
-   * @param {Vector3} c
-   * @returns {boolean} true if v is within triangle.
-   */
-  static testPointTriangle(v, a, b, c) {
-    function Test(_v, _a, _b) {
-      const crossA = _v.cross(_a)
-      const crossB = _v.cross(_b)
-      const dotWithEpsilon = Number.EPSILON + crossA.dot(crossB)
-      return Math.sign(dotWithEpsilon) !== -1
-    }
-
-    return (
-      Test(b.sub(a), v.sub(a), c.sub(a)) &&
-      Test(c.sub(b), v.sub(b), a.sub(b)) &&
-      Test(a.sub(c), v.sub(c), b.sub(c))
-    )
-  }
-
-  /**
-   * Checks that triangle abc is clockwise with reference to referenceNormal.
-   * @param {Vector3} referenceNormal The normal direction of the face.
-   * @param {Vector3} a
-   * @param {Vector3} b
-   * @param {Vector3} c
-   * @returns {boolean} true if triangle is ccw
-   */
-  static triangleIsCCW(referenceNormal, a, b, c) {
-    const triangleNormal = c.sub(a).cross(b.sub(a))
-    triangleNormal.normalize()
-    return referenceNormal.dot(triangleNormal) > 0.0
-  }
-}
-
-/**
- * Encapsulates vector maths operations required for polygon triangulation
- */
-class Vector3 {
-  constructor(x, y, z) {
-    this.x = x
-    this.y = y
-    this.z = z
-  }
-
-  add(v) {
-    return new Vector3(this.x + v.x, this.y + v.y, this.z + v.z)
-  }
-
-  sub(v) {
-    return new Vector3(this.x - v.x, this.y - v.y, this.z - v.z)
-  }
-
-  mul(n) {
-    return new Vector3(this.x - n, this.y - n, this.z - n)
-  }
-
-  dot(v) {
-    return this.x * v.x + this.y * v.y + this.z * v.z
-  }
-
-  cross(v) {
-    const nx = this.y * v.z - this.z * v.y
-    const ny = this.z * v.x - this.x * v.z
-    const nz = this.x * v.y - this.y * v.x
-
-    return new Vector3(nx, ny, nz)
-  }
-
-  squareSum() {
-    return this.x * this.x + this.y * this.y + this.z * this.z
-  }
-
-  normalize() {
-    const scale = 1.0 / Math.sqrt(this.squareSum())
-    this.x *= scale
-    this.y *= scale
-    this.z *= scale
+    return localOffset
   }
 }

packages/viewer/src/modules/converter/VirtualArray.ts

@@ -0,0 +1,177 @@
+import { TypedArray } from 'type-fest'
+import { DataChunk } from '../../IViewer.js'
+import { Box3, MathUtils, Vector3 } from 'three'
+
+export class VirtualArray {
+  private offsets: number[]
+
+  constructor(public chunks: Array<Array<number>>) {
+    this.updateOffsets()
+  }
+
+  get length() {
+    if (this.chunks.length === 0) return 0
+    const lastChunk = this.chunks[this.chunks.length - 1]
+    return this.offsets[this.offsets.length - 1] + lastChunk.length
+  }
+
+  get(index: number): number {
+    if (this.chunks.length === 1) return this.chunks[0][index]
+    const chunkIndex = this.findChunkIndex(index)
+    const localIndex = index - this.offsets[chunkIndex]
+    return this.chunks[chunkIndex][localIndex]
+  }
+
+  set(index: number, value: number) {
+    if (this.chunks.length === 1) {
+      this.chunks[0][index] = value
+      return
+    }
+    const chunkIndex = this.findChunkIndex(index)
+    const localIndex = index - this.offsets[chunkIndex]
+    this.chunks[chunkIndex][localIndex] = value
+  }
+
+  public findChunkIndex(index: number): number {
+    let low = 0
+    let high = this.offsets.length - 1
+
+    while (low <= high) {
+      const mid = (low + high) >> 1
+      const start = this.offsets[mid]
+      const end = mid + 1 < this.offsets.length ? this.offsets[mid + 1] : this.length
+      if (index >= start && index < end) return mid
+      if (index < start) high = mid - 1
+      else low = mid + 1
+    }
+
+    throw new RangeError('Index out of bounds')
+  }
+
+  public updateOffsets() {
+    this.offsets = []
+    let sum = 0
+    for (const chunk of this.chunks) {
+      this.offsets.push(sum)
+      sum += chunk.length
+    }
+  }
+}
+
+export class ChunkArray extends VirtualArray {
+  public chunkArray: Array<DataChunk>
+  protected flatArray: TypedArray
+
+  constructor(chunks: Array<DataChunk>) {
+    super(chunks && chunks.map((c: DataChunk) => c.data))
+    this.chunkArray = chunks
+  }
+
+  public slice() {
+    const copiesArray: Array<DataChunk> = []
+    this.chunkArray.forEach((chunk: DataChunk) => {
+      const chunkCopy = new Array<number>(chunk.data.length)
+      for (let k = 0; k < chunk.data.length; k++) {
+        chunkCopy[k] = chunk.data[k]
+      }
+      copiesArray.push({ data: chunkCopy, id: MathUtils.generateUUID(), references: 1 })
+    })
+    return new ChunkArray(copiesArray)
+  }
+
+  public copyToBuffer(buffer: TypedArray, offset: number) {
+    let chunkOffset = 0
+
+    this.chunkArray.forEach((chunk: DataChunk) => {
+      buffer.set(
+        chunk.data as unknown as ArrayLike<number> & ArrayLike<bigint>,
+        offset + chunkOffset
+      )
+      chunkOffset += chunk.data.length
+    })
+  }
+
+  public computeBox3(): Box3 {
+    const box = new Box3()
+    const vec3 = new Vector3()
+    let carry: number[] = [] // to hold x/y if vec3 is split
+
+    for (let c = 0; c < this.chunks.length; c++) {
+      const chunk = this.chunks[c]
+      let i = 0
+
+      // Handle carry-over from previous chunk
+      if (carry.length > 0) {
+        while (carry.length < 3 && i < chunk.length) {
+          carry.push(chunk[i++])
+        }
+        if (carry.length === 3) {
+          vec3.set(carry[0], carry[1], carry[2])
+          box.expandByPoint(vec3)
+          carry = []
+        }
+      }
+
+      // Now read as many full vec3s as possible from this chunk
+      const fullVec3Count = Math.floor((chunk.length - i) / 3)
+      for (let j = 0; j < fullVec3Count; j++) {
+        const x = chunk[i++]
+        const y = chunk[i++]
+        const z = chunk[i++]
+        vec3.set(x, y, z)
+        box.expandByPoint(vec3)
+      }
+
+      // If there's a leftover partial vec3 at the end, save it
+      while (i < chunk.length) {
+        carry.push(chunk[i++])
+      }
+    }
+
+    // Final sanity check
+    if (carry.length !== 0) {
+      console.warn('Virtual position buffer ended with incomplete vec3 data')
+    }
+
+    return box
+  }
+
+  protected getFlatArray<T extends TypedArray>(Type: { new (length: number): T }) {
+    if (!this.flatArray || !(this.flatArray instanceof Type)) {
+      this.flatArray = new Type(this.length)
+      let chunkOffset = 0
+      this.chunks.forEach((chunk: number[]) => {
+        this.flatArray.set(
+          chunk as unknown as ArrayLike<number> & ArrayLike<bigint>,
+          chunkOffset
+        )
+        chunkOffset += chunk.length
+      })
+    }
+    return this.flatArray as T
+  }
+
+  public getFloat32Array(): Float32Array {
+    return this.getFlatArray(Float32Array)
+  }
+
+  public getFloat64Array(): Float64Array {
+    return this.getFlatArray(Float64Array)
+  }
+
+  public getInt16Array(): Int16Array {
+    return this.getFlatArray(Int16Array)
+  }
+
+  public getInt32Array(): Int32Array {
+    return this.getFlatArray(Int32Array)
+  }
+
+  public getUint16Array(): Uint16Array {
+    return this.getFlatArray(Uint16Array)
+  }
+
+  public getUint32Array(): Uint32Array {
+    return this.getFlatArray(Uint32Array)
+  }
+}

packages/viewer/src/modules/extensions/TransformControls.js

@@ -1279,8 +1279,8 @@ class TransformControlsGizmo extends Object3D {
           [0, 0, 0],
           [0, 0, -Math.PI / 2]
         ]
-      ],
-      E: [[new Mesh(new TorusGeometry(0.75, 0.1, 2, 24), matInvisible)]]
+      ]
+      // E: [[new Mesh(new TorusGeometry(0.75, 0.1, 2, 24), matInvisible)]]
     }
 
     const gizmoScale = {

packages/viewer/src/modules/extensions/measurements/AreaMeasurement.ts

@@ -16,6 +16,7 @@ import {
   Quaternion,
   Raycaster,
   ReplaceStencilOp,
+  Uint16BufferAttribute,
   Vector2,
   Vector3,
   type Intersection
@@ -370,7 +371,7 @@ export class AreaMeasurement extends Measurement {
     }
 
     if (!index || index.count !== indices.length) {
-      geometry.setIndex(new BufferAttribute(new Uint16Array(indices), 1))
+      geometry.setIndex(new Uint16BufferAttribute(indices, 1))
     } else {
       ;(index.array as Uint16Array).set(indices, 0)
       index.needsUpdate = true

packages/viewer/src/modules/extensions/sections/SectionOutlines.ts

@@ -303,6 +303,7 @@ export class SectionOutlines extends Extension {
   private createPlaneOutline(planeId: string): PlaneOutline {
     const buffer = new Float64Array(SectionOutlines.INITIAL_BUFFER_SIZE)
     const lineGeometry = new LineSegmentsGeometry()
+    /** We need to re-allocate, otherwise three.js will do it anyway */
     lineGeometry.setPositions(new Float32Array(buffer))
     ;(
       lineGeometry.attributes['instanceStart'] as InterleavedBufferAttribute
@@ -390,7 +391,7 @@ export class SectionOutlines extends Extension {
 
     const buffer = new Float32Array(size)
     outline.renderable.geometry = new LineSegmentsGeometry()
-    outline.renderable.geometry.setPositions(new Float32Array(buffer))
+    outline.renderable.geometry.setPositions(buffer)
     ;(
       outline.renderable.geometry.attributes[
         'instanceStart'

packages/viewer/src/modules/extensions/sections/SectionTool.ts

@@ -21,6 +21,8 @@ import {
   Color,
   MeshBasicMaterial,
   PlaneGeometry,
+  Float32BufferAttribute,
+  Uint16BufferAttribute,
   Euler
 } from 'three'
 import { intersectObjectWithRay, TransformControls } from '../TransformControls.js'
@@ -61,7 +63,7 @@ const _vector3 = new Vector3()
 const _tempEuler = new Euler()
 const _tempQuaternion = new Quaternion()
 
-const unitCube = [
+const unitCube = new Float32Array([
   -1 * 0.5,
   -1 * 0.5,
   -1 * 0.5,
@@ -93,12 +95,12 @@ const unitCube = [
   -1 * 0.5,
   1 * 0.5,
   1 * 0.5
-]
+])
 
-const unitCubeIndices: number[] = [
+const unitCubeIndices: Uint16Array = new Uint16Array([
   0, 1, 3, 3, 1, 2, 1, 5, 2, 2, 5, 6, 5, 4, 6, 6, 4, 7, 4, 0, 7, 7, 0, 3, 3, 2, 7, 7, 2,
   6, 4, 5, 0, 0, 5, 1
-]
+])
 
 const unitCubeEdges: number[] = [
   // Bottom Face
@@ -932,11 +934,11 @@ export class SectionTool extends Extension {
   /** Creates the geometry for the visible outline of the section tool */
   protected createOutline() {
     /** We start from the unit cube's edges */
-    const buffer = new Float32Array(unitCubeEdges.slice())
+    const buffer = unitCubeEdges.slice() as unknown as Float32Array
 
     /** Create the line segments geometry */
     const lineGeometry = new LineSegmentsGeometry()
-    lineGeometry.setPositions(new Float32Array(buffer))
+    lineGeometry.setPositions(buffer)
     ;(
       lineGeometry.attributes['instanceStart'] as InterleavedBufferAttribute
     ).data.setUsage(DynamicDrawUsage)
@@ -1015,8 +1017,8 @@ export class SectionTool extends Extension {
     const indexes = unitCubeIndices.slice()
 
     const g = new BufferGeometry()
-    g.setAttribute('position', new BufferAttribute(new Float32Array(vertices), 3))
-    g.setIndex(indexes)
+    g.setAttribute('position', new Float32BufferAttribute(vertices, 3))
+    g.setIndex(new Uint16BufferAttribute(indexes, 1))
     g.computeBoundingBox()
     g.computeVertexNormals()
     return g

packages/viewer/src/modules/loaders/Speckle/SpeckleConverter.ts

@@ -6,6 +6,7 @@ import { SpeckleType, type SpeckleObject } from '../../../index.js'
 import Logger from '../../utils/Logger.js'
 import { ObjectLoader2 } from '@speckle/objectloader2'
 import { SpeckleTypeAllRenderables } from '../GeometryConverter.js'
+import { DataChunk } from '../../../IViewer.js'
 
 export type ConverterResultDelegate = (count: number) => void
 export type SpeckleConverterNodeDelegate =
@@ -64,7 +65,7 @@ export default class SpeckleConverter {
     Parameter: null
   }
 
-  protected readonly IgnoreNodes = ['Parameter']
+  protected readonly IgnoreNodes = ['Parameter', 'RawEncoding']
 
   constructor(objectLoader: ObjectLoader2, tree: WorldTree) {
     if (!objectLoader) {
@@ -287,24 +288,49 @@ export default class SpeckleConverter {
    * @param  {[type]} arr [description]
    * @return {[type]}     [description]
    */
-  private async dechunk(arr: Array<{ referencedId: string }>) {
-    if (!arr || arr.length === 0) return arr
-    // Handles pre-chunking objects, or arrs that have not been chunked
-    if (!arr[0].referencedId) return arr
+  private async dechunk(arr: Array<{ referencedId: string }>): Promise<DataChunk[]> {
+    if (!arr || arr.length === 0) {
+      return arr as unknown as DataChunk[]
+    }
 
-    const chunked: unknown[] = []
+    if (Array.isArray(arr[0]) && !arr[0].referencedId) {
+      return arr as unknown as DataChunk[]
+    }
+    // Handles pre-chunking objects, or arrs that have not been chunked
+    if (!arr[0].referencedId) {
+      if (!(arr[0] instanceof Object))
+        return [
+          {
+            data: arr,
+            id: MathUtils.generateUUID(),
+            references: 1
+          }
+        ] as unknown as DataChunk[]
+      else return arr as unknown as DataChunk[]
+    }
+
+    const chunked: DataChunk[] = []
     for (const ref of arr) {
-      const real: Record<string, unknown> = (await this.objectLoader.getObject({
+      const real: DataChunk = (await this.objectLoader.getObject({
         id: ref.referencedId
-      })) as unknown as Record<string, number>
-      chunked.push(real.data)
+      })) as unknown as DataChunk
+      if (real.references === undefined) {
+        real.references = 1
+      } else {
+        real.references++
+      }
+      if (typeof real.data[0] !== 'number' || isNaN(real.data[0])) {
+        Logger.error(
+          `Chunk id ${real.id} used for mesh ${ref.referencedId} might not have numeric geometry data. This is not supported!`
+        )
+      }
+      chunked.push(real)
       // await this.asyncPause()
     }
 
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const dechunked = [].concat(...(chunked as any))
+    // const dechunked = [].concat(...(chunked as any))
 
-    return dechunked
+    return chunked
   }
 
   /**
@@ -916,14 +942,23 @@ export default class SpeckleConverter {
       Logger.warn(
         `Object id ${obj.id} of type ${obj.speckle_type} has no vertex position data and will be ignored`
       )
+      node.model.raw.vertices = []
+      node.model.raw.faces = []
+      node.model.raw.colors = []
+      node.model.raw.vertexNormals = []
       return
     }
     if (!obj.faces || (obj.faces as Array<number>).length === 0) {
       Logger.warn(
         `Object id ${obj.id} of type ${obj.speckle_type} has no face data and will be ignored`
       )
+      node.model.raw.vertices = []
+      node.model.raw.faces = []
+      node.model.raw.colors = []
+      node.model.raw.vertexNormals = []
       return
     }
+
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     //@ts-ignore
     node.model.raw.vertices = await this.dechunk(obj.vertices)

packages/viewer/src/modules/loaders/Speckle/SpeckleGeometryConverter.ts

@@ -2,9 +2,11 @@ import { Geometry, type GeometryData } from '../../converter/Geometry.js'
 import MeshTriangulationHelper from '../../converter/MeshTriangulationHelper.js'
 import { getConversionFactor } from '../../converter/Units.js'
 import { type NodeData } from '../../tree/WorldTree.js'
-import { Box3, EllipseCurve, Matrix4, Vector2, Vector3 } from 'three'
+import { Box3, EllipseCurve, MathUtils, Matrix4, Vector2, Vector3 } from 'three'
 import { GeometryConverter, SpeckleType } from '../GeometryConverter.js'
 import Logger from '../../utils/Logger.js'
+import { DataChunk } from '../../../IViewer.js'
+import { ChunkArray } from '../../converter/VirtualArray.js'
 
 export class SpeckleGeometryConverter extends GeometryConverter {
   public typeLookupTable: { [type: string]: SpeckleType } = {}
@@ -93,9 +95,35 @@ export class SpeckleGeometryConverter extends GeometryConverter {
         node.raw.colors = []
         break
       case SpeckleType.Mesh:
+        /** Raw objects will no longer hold references to chunks */
         node.raw.vertices = []
         node.raw.faces = []
         node.raw.colors = []
+        node.raw.normals = []
+
+        // /** We can already delete these because we don't need them after triangulation */
+        // node.raw.faces.forEach((c: DataChunk) => {
+        //   c.references--
+
+        //   if (!c.references) {
+        //     Logger.warn(`Deleting chunk data ${c.id}`)
+        //     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        //     //@ts-ignore
+        //     delete c.data
+        //   }
+        // })
+
+        /** We can already delete this because we've changes the colors to floats in linear space */
+        node.raw.colors.forEach((c: DataChunk) => {
+          c.references--
+
+          if (!c.references) {
+            Logger.warn(`Deleting chunk data ${c.id}`)
+            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+            //@ts-ignore
+            delete c.data
+          }
+        })
         break
       case SpeckleType.Point:
         if (node.raw.value) node.raw.value = []
@@ -197,8 +225,10 @@ export class SpeckleGeometryConverter extends GeometryConverter {
   protected PointcloudToGeometryData(node: NodeData): GeometryData | null {
     const conversionFactor = getConversionFactor(node.raw.units)
 
-    const vertices = node.instanced ? node.raw.points.slice() : node.raw.points
-    const colorsRaw = node.raw.colors
+    const vertices = new ChunkArray(
+      node.instanced ? node.raw.points.slice() : node.raw.points
+    )
+    const colorsRaw = new ChunkArray(node.raw.colors)
     let colors = null
 
     if (colorsRaw && colorsRaw.length !== 0) {
@@ -215,6 +245,10 @@ export class SpeckleGeometryConverter extends GeometryConverter {
       attributes: {
         POSITION: vertices,
         COLOR: colors
+          ? new ChunkArray([
+              { data: colors, id: MathUtils.generateUUID(), references: 1 }
+            ])
+          : undefined
       },
       bakeTransform: new Matrix4().makeScale(
         conversionFactor,
@@ -254,45 +288,101 @@ export class SpeckleGeometryConverter extends GeometryConverter {
     if (!node.raw) return null
 
     const conversionFactor = getConversionFactor(node.raw.units)
-    const indices = []
 
     if (!node.raw.vertices) return null
     if (!node.raw.faces) return null
 
     const start = performance.now()
-    const vertices = node.raw.vertices
-    const faces = node.raw.faces
-    const colorsRaw = node.raw.colors
+
+    const vertices = new ChunkArray(node.raw.vertices)
+    const faces = new ChunkArray(node.raw.faces)
+    const colorsRaw = this.chunkArrayHasData(node.raw.colors)
+      ? new ChunkArray(node.raw.colors)
+      : undefined
     let normals = node.raw.vertexNormals
+      ? new ChunkArray(node.raw.vertexNormals)
+      : undefined
     let colors = undefined
     let k = 0
+    let triangulated = true
+    let triangulatedArraySize = 0
     while (k < faces.length) {
-      let n = faces[k]
+      const chunkIndex = faces.findChunkIndex(k)
+      if (faces.chunkArray[chunkIndex].processed) {
+        k += faces.chunkArray[chunkIndex].data.length
+        continue
+      }
+      let n = faces.get(k)
       if (n < 3) n += 3 // 0 -> 3, 1 -> 4
+      k += n + 1
 
       if (n === 3) {
-        const startP = performance.now()
-        // Triangle face
-        indices.push(faces[k + 1], faces[k + 2], faces[k + 3])
-        this.pushTime += performance.now() - startP
+        triangulatedArraySize += 3
+        continue
       } else {
-        // Quad or N-gon face
-        const start1 = performance.now()
-        const triangulation = MeshTriangulationHelper.triangulateFace(
-          k,
-          faces,
-          vertices
-        )
-        this.actualTriangulateTime += performance.now() - start1
-        indices.push(
-          ...triangulation.filter((el) => {
-            return el !== undefined
-          })
-        )
+        triangulatedArraySize += (n - 2) * 3
+        triangulated = false
       }
-
-      k += n + 1
     }
+
+    const indices =
+      triangulatedArraySize >= 65535 || vertices.length >= 65535
+        ? new Uint32Array(triangulatedArraySize)
+        : new Uint16Array(triangulatedArraySize)
+    let indicesOffset = 0
+
+    if (triangulated) {
+      /** If already triangulated modfy the faces array in place */
+      faces.chunkArray.forEach((chunk: DataChunk) => {
+        if (chunk.processed) return
+
+        let write = 0
+        for (let read = 0; read < chunk.data.length; read++) {
+          if (read % 4 !== 0) {
+            chunk.data[write++] = chunk.data[read]
+          }
+        }
+        chunk.data.length = write
+        chunk.processed = true
+      })
+      faces.updateOffsets()
+    } else {
+      k = 0
+      while (k < faces.length) {
+        /** We skip to the end of triangulated chunks */
+        const chunkIndex = faces.findChunkIndex(k)
+        if (faces.chunkArray[chunkIndex].processed) {
+          indices.set(faces.chunks[chunkIndex], k)
+          k += faces.chunkArray[chunkIndex].data.length
+          continue
+        }
+        let n = faces.get(k)
+        if (n < 3) n += 3 // 0 -> 3, 1 -> 4
+        if (n === 3) {
+          // Triangle face
+          indices[indicesOffset] = faces.get(k + 1)
+          indices[indicesOffset + 1] = faces.get(k + 2)
+          indices[indicesOffset + 2] = faces.get(k + 3)
+          indicesOffset += 3
+        } else {
+          const start1 = performance.now()
+          const indexCount = MeshTriangulationHelper.triangulateFace(
+            k,
+            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+            //@ts-ignore
+            faces,
+            vertices,
+            indices, // inout
+            indicesOffset // in
+          )
+          indicesOffset += indexCount
+          this.actualTriangulateTime += performance.now() - start1
+        }
+
+        k += n + 1
+      }
+    }
+
     this.meshTriangulationTime += performance.now() - start
 
     if (colorsRaw && colorsRaw.length !== 0) {
@@ -302,7 +392,13 @@ export class SpeckleGeometryConverter extends GeometryConverter {
         )
       } else
       /** We want the colors in linear space */
-        colors = this.unpackColors(colorsRaw, true)
+        colors = new ChunkArray([
+          {
+            id: MathUtils.generateUUID(),
+            references: 1,
+            data: this.unpackColors(colorsRaw, true)
+          }
+        ])
     }
 
     if (normals && normals.length !== 0) {
@@ -312,12 +408,22 @@ export class SpeckleGeometryConverter extends GeometryConverter {
         )
         normals = undefined
       }
-    } else normals = undefined
+    } else {
+      normals = undefined
+    }
 
     return {
       attributes: {
         POSITION: vertices,
-        INDEX: indices,
+        INDEX: triangulated
+          ? faces
+          : new ChunkArray([
+              {
+                data: indices as unknown as number[],
+                id: MathUtils.generateUUID(),
+                references: 1
+              }
+            ]),
         ...(colors && { COLOR: colors }),
         ...(normals && { NORMAL: normals })
       },
@@ -368,15 +474,18 @@ export class SpeckleGeometryConverter extends GeometryConverter {
    */
   protected PointToGeometryData(node: NodeData): GeometryData | null {
     const conversionFactor = getConversionFactor(node.raw.units)
+    const points = this.PointToFloatArray(
+      node.raw as { value: Array<number>; units: string } & {
+        x: number
+        y: number
+        z: number
+      }
+    )
     return {
       attributes: {
-        POSITION: this.PointToFloatArray(
-          node.raw as { value: Array<number>; units: string } & {
-            x: number
-            y: number
-            z: number
-          }
-        )
+        POSITION: new ChunkArray([
+          { data: points, id: MathUtils.generateUUID(), references: 1 }
+        ])
       },
       bakeTransform: new Matrix4().makeScale(
         conversionFactor,
@@ -394,9 +503,15 @@ export class SpeckleGeometryConverter extends GeometryConverter {
     const conversionFactor = getConversionFactor(node.raw.units)
     return {
       attributes: {
-        POSITION: this.PointToFloatArray(node.raw.start).concat(
-          this.PointToFloatArray(node.raw.end)
-        )
+        POSITION: new ChunkArray([
+          {
+            data: this.PointToFloatArray(node.raw.start).concat(
+              this.PointToFloatArray(node.raw.end)
+            ),
+            id: MathUtils.generateUUID(),
+            references: 1
+          }
+        ])
       },
       bakeTransform: new Matrix4().makeScale(
         conversionFactor,
@@ -412,12 +527,26 @@ export class SpeckleGeometryConverter extends GeometryConverter {
    */
   protected PolylineToGeometryData(node: NodeData): GeometryData | null {
     const conversionFactor = getConversionFactor(node.raw.units)
+    const chunkArray = new ChunkArray(node.raw.value)
 
-    if (node.raw.closed)
-      node.raw.value.push(node.raw.value[0], node.raw.value[1], node.raw.value[2])
+    let outChunk = chunkArray
+    if (node.raw.closed) {
+      const complete = new Float32Array(chunkArray.length + 3)
+      chunkArray.copyToBuffer(complete, 0)
+      complete[chunkArray.length] = complete[0]
+      complete[chunkArray.length + 1] = complete[1]
+      complete[chunkArray.length + 2] = complete[2]
+      outChunk = new ChunkArray([
+        {
+          data: complete as unknown as number[],
+          id: MathUtils.generateUUID(),
+          references: 1
+        }
+      ])
+    }
     return {
       attributes: {
-        POSITION: node.raw.value.slice(0)
+        POSITION: outChunk
       },
       bakeTransform: new Matrix4().makeScale(
         conversionFactor,
@@ -504,7 +633,9 @@ export class SpeckleGeometryConverter extends GeometryConverter {
 
     return {
       attributes: {
-        POSITION: edges
+        POSITION: new ChunkArray([
+          { data: edges, id: MathUtils.generateUUID(), references: 1 }
+        ])
       },
       bakeTransform: new Matrix4().copy(T).multiply(R),
       transform: null
@@ -562,7 +693,13 @@ export class SpeckleGeometryConverter extends GeometryConverter {
     )
     return {
       attributes: {
-        POSITION: this.FlattenVector3Array(points)
+        POSITION: new ChunkArray([
+          {
+            data: this.FlattenVector3Array(points),
+            id: MathUtils.generateUUID(),
+            references: 1
+          }
+        ])
       },
       bakeTransform: null,
       transform: null
@@ -664,7 +801,13 @@ export class SpeckleGeometryConverter extends GeometryConverter {
 
     return {
       attributes: {
-        POSITION: this.FlattenVector3Array(points)
+        POSITION: new ChunkArray([
+          {
+            data: this.FlattenVector3Array(points),
+            id: MathUtils.generateUUID(),
+            references: 1
+          }
+        ])
       },
       bakeTransform: matrix,
       transform: null
@@ -710,7 +853,13 @@ export class SpeckleGeometryConverter extends GeometryConverter {
 
     return {
       attributes: {
-        POSITION: this.FlattenVector3Array(points)
+        POSITION: new ChunkArray([
+          {
+            data: this.FlattenVector3Array(points),
+            id: MathUtils.generateUUID(),
+            references: 1
+          }
+        ])
       },
       bakeTransform: null,
       transform: null
@@ -818,10 +967,10 @@ export class SpeckleGeometryConverter extends GeometryConverter {
     return output
   }
 
-  protected unpackColors(int32Colors: number[], tolinear = false): number[] {
+  protected unpackColors(int32Colors: ChunkArray, tolinear = false): number[] {
     const colors = new Array<number>(int32Colors.length * 3)
     for (let i = 0; i < int32Colors.length; i++) {
-      const color = int32Colors[i]
+      const color = int32Colors.get(i)
       const r = (color >> 16) & 0xff
       const g = (color >> 8) & 0xff
       const b = color & 0xff
@@ -843,4 +992,9 @@ export class SpeckleGeometryConverter extends GeometryConverter {
     else if (x < 0.04045) return x / 12.92
     else return Math.pow((x + 0.055) / 1.055, 2.4)
   }
+
+  /** Connectors send empty chunks ಠ_ಠ */
+  protected chunkArrayHasData(chunks: Array<DataChunk>): boolean {
+    return chunks && chunks.filter((c: DataChunk) => c.data && c.data.length).length > 0
+  }
 }

packages/viewer/src/modules/materials/Materials.ts

@@ -15,6 +15,7 @@ import { SpeckleMaterial } from './SpeckleMaterial.js'
 import SpecklePointColouredMaterial from './SpecklePointColouredMaterial.js'
 import { type Asset, AssetType, type MaterialOptions } from '../../IViewer.js'
 import SpeckleTextColoredMaterial from './SpeckleTextColoredMaterial.js'
+import { ChunkArray } from '../converter/VirtualArray.js'
 
 const defaultGradient: Asset = {
   id: 'defaultGradient',
@@ -124,6 +125,10 @@ export default class Materials {
     if (!materialNode) return null
     let renderMaterial: RenderMaterial | null = null
     if (materialNode.model.raw.renderMaterial) {
+      const colorsChunkArray = geometryNode?.model.raw.colors
+        ? new ChunkArray(geometryNode?.model.raw.colors)
+        : undefined
+
       renderMaterial = {
         id: materialNode.model.raw.renderMaterial.id,
         color: materialNode.model.raw.renderMaterial.diffuse,
@@ -135,9 +140,7 @@ export default class Materials {
         roughness: materialNode.model.raw.renderMaterial.roughness,
         metalness: materialNode.model.raw.renderMaterial.metalness,
         vertexColors:
-          geometryNode &&
-          geometryNode.model.raw.colors &&
-          geometryNode.model.raw.colors.length > 0
+          (geometryNode && colorsChunkArray && colorsChunkArray.length > 0) ?? false
       }
     }
     return renderMaterial
@@ -273,7 +276,8 @@ export default class Materials {
       colorMaterialData &&
       !materialData &&
       (renderView.geometryType === GeometryType.LINE ||
-        renderView.geometryType === GeometryType.POINT)
+        renderView.geometryType === GeometryType.POINT ||
+        renderView.geometryType === GeometryType.TEXT)
 
     const displayStyleFirst = renderView.geometryType === GeometryType.LINE
     if (!materialData) {

packages/viewer/src/modules/materials/shaders/speckle-basic-vert.ts

@@ -149,7 +149,32 @@ void main() {
 		#include <morphnormal_vertex>
 		#include <skinbase_vertex>
 		#include <skinnormal_vertex>
-		#include <defaultnormal_vertex>
+        // #include <defaultnormal_vertex> // COMMENTED CHUNK
+        vec3 transformedNormal = objectNormal;
+        #ifdef USE_INSTANCING
+
+            // this is in lieu of a per-instance normal-matrix
+            // shear transforms in the instance matrix are not supported
+            mat3 m = mat3( instanceMatrix );
+            transformedNormal /= vec3( dot( m[ 0 ], m[ 0 ] ), dot( m[ 1 ], m[ 1 ] ), dot( m[ 2 ], m[ 2 ] ) );
+            transformedNormal = m * transformedNormal;
+            
+            /* If we have negative scaling, we flip the normal */
+            float signDet = sign(dot(m[0], cross(m[1], m[2])));
+            // Optional fallback: treat 0 as +1
+            signDet = signDet + (1.0 - abs(signDet));
+            transformedNormal *= signDet;
+        #endif
+        transformedNormal = normalMatrix * transformedNormal;
+        #ifdef FLIP_SIDED
+            transformedNormal = - transformedNormal;
+        #endif
+        #ifdef USE_TANGENT
+            vec3 transformedTangent = ( modelViewMatrix * vec4( objectTangent, 0.0 ) ).xyz;
+            #ifdef FLIP_SIDED
+                transformedTangent = - transformedTangent;
+            #endif
+        #endif
 	#endif
 	#include <begin_vertex>
 	#include <morphtarget_vertex>

packages/viewer/src/modules/materials/shaders/speckle-depth-normal-id-vert.ts

@@ -205,7 +205,32 @@ void main() {
 		#include <morphnormal_vertex>
 		#include <skinnormal_vertex>
 	#endif
-    #include <defaultnormal_vertex>
+    // #include <defaultnormal_vertex> // COMMENTED CHUNK
+    vec3 transformedNormal = objectNormal;
+    #ifdef USE_INSTANCING
+
+        // this is in lieu of a per-instance normal-matrix
+        // shear transforms in the instance matrix are not supported
+        mat3 m = mat3( instanceMatrix );
+        transformedNormal /= vec3( dot( m[ 0 ], m[ 0 ] ), dot( m[ 1 ], m[ 1 ] ), dot( m[ 2 ], m[ 2 ] ) );
+        transformedNormal = m * transformedNormal;
+        
+        /* If we have negative scaling, we flip the normal */
+        float signDet = sign(dot(m[0], cross(m[1], m[2])));
+        // Optional fallback: treat 0 as +1
+        signDet = signDet + (1.0 - abs(signDet));
+        transformedNormal *= signDet;
+    #endif
+    transformedNormal = normalMatrix * transformedNormal;
+    #ifdef FLIP_SIDED
+        transformedNormal = - transformedNormal;
+    #endif
+    #ifdef USE_TANGENT
+        vec3 transformedTangent = ( modelViewMatrix * vec4( objectTangent, 0.0 ) ).xyz;
+        #ifdef FLIP_SIDED
+            transformedTangent = - transformedTangent;
+        #endif
+    #endif
 	#include <normal_vertex>
 	#include <begin_vertex>
 	#include <morphtarget_vertex>

packages/viewer/src/modules/materials/shaders/speckle-depth-normal-vert.ts

@@ -146,7 +146,32 @@ void main() {
 		#include <morphnormal_vertex>
 		#include <skinnormal_vertex>
 	#endif
-    #include <defaultnormal_vertex>
+    // #include <defaultnormal_vertex> // COMMENTED CHUNK
+    vec3 transformedNormal = objectNormal;
+    #ifdef USE_INSTANCING
+
+        // this is in lieu of a per-instance normal-matrix
+        // shear transforms in the instance matrix are not supported
+        mat3 m = mat3( instanceMatrix );
+        transformedNormal /= vec3( dot( m[ 0 ], m[ 0 ] ), dot( m[ 1 ], m[ 1 ] ), dot( m[ 2 ], m[ 2 ] ) );
+        transformedNormal = m * transformedNormal;
+        
+        /* If we have negative scaling, we flip the normal */
+        float signDet = sign(dot(m[0], cross(m[1], m[2])));
+        // Optional fallback: treat 0 as +1
+        signDet = signDet + (1.0 - abs(signDet));
+        transformedNormal *= signDet;
+    #endif
+    transformedNormal = normalMatrix * transformedNormal;
+    #ifdef FLIP_SIDED
+        transformedNormal = - transformedNormal;
+    #endif
+    #ifdef USE_TANGENT
+        vec3 transformedTangent = ( modelViewMatrix * vec4( objectTangent, 0.0 ) ).xyz;
+        #ifdef FLIP_SIDED
+            transformedTangent = - transformedTangent;
+        #endif
+    #endif
 	#include <normal_vertex>
 	#include <begin_vertex>
 	#include <morphtarget_vertex>

packages/viewer/src/modules/materials/shaders/speckle-displace.vert.ts

@@ -143,7 +143,32 @@ void main() {
 		#include <morphnormal_vertex>
 		#include <skinbase_vertex>
 		#include <skinnormal_vertex>
-		#include <defaultnormal_vertex>
+        // #include <defaultnormal_vertex> // COMMENTED CHUNK
+        vec3 transformedNormal = objectNormal;
+        #ifdef USE_INSTANCING
+
+            // this is in lieu of a per-instance normal-matrix
+            // shear transforms in the instance matrix are not supported
+            mat3 m = mat3( instanceMatrix );
+            transformedNormal /= vec3( dot( m[ 0 ], m[ 0 ] ), dot( m[ 1 ], m[ 1 ] ), dot( m[ 2 ], m[ 2 ] ) );
+            transformedNormal = m * transformedNormal;
+            
+            /* If we have negative scaling, we flip the normal */
+            float signDet = sign(dot(m[0], cross(m[1], m[2])));
+            // Optional fallback: treat 0 as +1
+            signDet = signDet + (1.0 - abs(signDet));
+            transformedNormal *= signDet;
+        #endif
+        transformedNormal = normalMatrix * transformedNormal;
+        #ifdef FLIP_SIDED
+            transformedNormal = - transformedNormal;
+        #endif
+        #ifdef USE_TANGENT
+            vec3 transformedTangent = ( modelViewMatrix * vec4( objectTangent, 0.0 ) ).xyz;
+            #ifdef FLIP_SIDED
+                transformedTangent = - transformedTangent;
+            #endif
+        #endif
 	#endif
 	#include <begin_vertex>
 	#include <morphtarget_vertex>

packages/viewer/src/modules/materials/shaders/speckle-ghost-vert.ts

@@ -140,7 +140,32 @@ void main() {
 		#include <morphnormal_vertex>
 		#include <skinbase_vertex>
 		#include <skinnormal_vertex>
-		#include <defaultnormal_vertex>
+        // #include <defaultnormal_vertex> // COMMENTED CHUNK
+        vec3 transformedNormal = objectNormal;
+        #ifdef USE_INSTANCING
+
+            // this is in lieu of a per-instance normal-matrix
+            // shear transforms in the instance matrix are not supported
+            mat3 m = mat3( instanceMatrix );
+            transformedNormal /= vec3( dot( m[ 0 ], m[ 0 ] ), dot( m[ 1 ], m[ 1 ] ), dot( m[ 2 ], m[ 2 ] ) );
+            transformedNormal = m * transformedNormal;
+            
+            /* If we have negative scaling, we flip the normal */
+            float signDet = sign(dot(m[0], cross(m[1], m[2])));
+            // Optional fallback: treat 0 as +1
+            signDet = signDet + (1.0 - abs(signDet));
+            transformedNormal *= signDet;
+        #endif
+        transformedNormal = normalMatrix * transformedNormal;
+        #ifdef FLIP_SIDED
+            transformedNormal = - transformedNormal;
+        #endif
+        #ifdef USE_TANGENT
+            vec3 transformedTangent = ( modelViewMatrix * vec4( objectTangent, 0.0 ) ).xyz;
+            #ifdef FLIP_SIDED
+                transformedTangent = - transformedTangent;
+            #endif
+        #endif
 	#endif
 	#include <begin_vertex>
 	#include <morphtarget_vertex>

packages/viewer/src/modules/materials/shaders/speckle-normal-vert.ts

@@ -139,7 +139,32 @@ void main() {
 	#include <morphnormal_vertex>
 	#include <skinbase_vertex>
 	#include <skinnormal_vertex>
-	#include <defaultnormal_vertex>
+    // #include <defaultnormal_vertex> // COMMENTED CHUNK
+    vec3 transformedNormal = objectNormal;
+    #ifdef USE_INSTANCING
+
+        // this is in lieu of a per-instance normal-matrix
+        // shear transforms in the instance matrix are not supported
+        mat3 m = mat3( instanceMatrix );
+        transformedNormal /= vec3( dot( m[ 0 ], m[ 0 ] ), dot( m[ 1 ], m[ 1 ] ), dot( m[ 2 ], m[ 2 ] ) );
+        transformedNormal = m * transformedNormal;
+        
+        /* If we have negative scaling, we flip the normal */
+        float signDet = sign(dot(m[0], cross(m[1], m[2])));
+        // Optional fallback: treat 0 as +1
+        signDet = signDet + (1.0 - abs(signDet));
+        transformedNormal *= signDet;
+    #endif
+    transformedNormal = normalMatrix * transformedNormal;
+    #ifdef FLIP_SIDED
+        transformedNormal = - transformedNormal;
+    #endif
+    #ifdef USE_TANGENT
+        vec3 transformedTangent = ( modelViewMatrix * vec4( objectTangent, 0.0 ) ).xyz;
+        #ifdef FLIP_SIDED
+            transformedTangent = - transformedTangent;
+        #endif
+    #endif
 	#include <normal_vertex>
 	#include <begin_vertex>
 	#include <morphtarget_vertex>

packages/viewer/src/modules/materials/shaders/speckle-standard-colored-vert.ts

@@ -158,7 +158,32 @@ void main() {
     #include <morphnormal_vertex>
     #include <skinbase_vertex>
     #include <skinnormal_vertex>
-    #include <defaultnormal_vertex>
+    // #include <defaultnormal_vertex> // COMMENTED CHUNK
+    vec3 transformedNormal = objectNormal;
+    #ifdef USE_INSTANCING
+
+        // this is in lieu of a per-instance normal-matrix
+        // shear transforms in the instance matrix are not supported
+        mat3 m = mat3( instanceMatrix );
+        transformedNormal /= vec3( dot( m[ 0 ], m[ 0 ] ), dot( m[ 1 ], m[ 1 ] ), dot( m[ 2 ], m[ 2 ] ) );
+        transformedNormal = m * transformedNormal;
+        
+        /* If we have negative scaling, we flip the normal */
+        float signDet = sign(dot(m[0], cross(m[1], m[2])));
+        // Optional fallback: treat 0 as +1
+        signDet = signDet + (1.0 - abs(signDet));
+        transformedNormal *= signDet;
+    #endif
+    transformedNormal = normalMatrix * transformedNormal;
+    #ifdef FLIP_SIDED
+        transformedNormal = - transformedNormal;
+    #endif
+    #ifdef USE_TANGENT
+        vec3 transformedTangent = ( modelViewMatrix * vec4( objectTangent, 0.0 ) ).xyz;
+        #ifdef FLIP_SIDED
+            transformedTangent = - transformedTangent;
+        #endif
+    #endif
     #include <normal_vertex>
 
     #include <begin_vertex>

packages/viewer/src/modules/materials/shaders/speckle-standard-vert.ts

@@ -155,7 +155,32 @@ void main() {
     #include <morphnormal_vertex>
     #include <skinbase_vertex>
     #include <skinnormal_vertex>
-    #include <defaultnormal_vertex>
+    // #include <defaultnormal_vertex> // COMMENTED CHUNK
+    vec3 transformedNormal = objectNormal;
+    #ifdef USE_INSTANCING
+
+        // this is in lieu of a per-instance normal-matrix
+        // shear transforms in the instance matrix are not supported
+        mat3 m = mat3( instanceMatrix );
+        transformedNormal /= vec3( dot( m[ 0 ], m[ 0 ] ), dot( m[ 1 ], m[ 1 ] ), dot( m[ 2 ], m[ 2 ] ) );
+        transformedNormal = m * transformedNormal;
+        
+        /* If we have negative scaling, we flip the normal */
+        float signDet = sign(dot(m[0], cross(m[1], m[2])));
+        // Optional fallback: treat 0 as +1
+        signDet = signDet + (1.0 - abs(signDet));
+        transformedNormal *= signDet;
+    #endif
+    transformedNormal = normalMatrix * transformedNormal;
+    #ifdef FLIP_SIDED
+        transformedNormal = - transformedNormal;
+    #endif
+    #ifdef USE_TANGENT
+        vec3 transformedTangent = ( modelViewMatrix * vec4( objectTangent, 0.0 ) ).xyz;
+        #ifdef FLIP_SIDED
+            transformedTangent = - transformedTangent;
+        #endif
+    #endif
     #include <normal_vertex>
 
     #include <begin_vertex>

packages/viewer/src/modules/materials/shaders/speckle-text-vert.ts

@@ -63,7 +63,32 @@ void main() {
 		#include <morphnormal_vertex>
 		#include <skinbase_vertex>
 		#include <skinnormal_vertex>
-		#include <defaultnormal_vertex>
+        // #include <defaultnormal_vertex> // COMMENTED CHUNK
+        vec3 transformedNormal = objectNormal;
+        #ifdef USE_INSTANCING
+
+            // this is in lieu of a per-instance normal-matrix
+            // shear transforms in the instance matrix are not supported
+            mat3 m = mat3( instanceMatrix );
+            transformedNormal /= vec3( dot( m[ 0 ], m[ 0 ] ), dot( m[ 1 ], m[ 1 ] ), dot( m[ 2 ], m[ 2 ] ) );
+            transformedNormal = m * transformedNormal;
+            
+            /* If we have negative scaling, we flip the normal */
+            float signDet = sign(dot(m[0], cross(m[1], m[2])));
+            // Optional fallback: treat 0 as +1
+            signDet = signDet + (1.0 - abs(signDet));
+            transformedNormal *= signDet;
+        #endif
+        transformedNormal = normalMatrix * transformedNormal;
+        #ifdef FLIP_SIDED
+            transformedNormal = - transformedNormal;
+        #endif
+        #ifdef USE_TANGENT
+            vec3 transformedTangent = ( modelViewMatrix * vec4( objectTangent, 0.0 ) ).xyz;
+            #ifdef FLIP_SIDED
+                transformedTangent = - transformedTangent;
+            #endif
+        #endif
 	#endif
 	#include <begin_vertex>
 	#include <morphtarget_vertex>

packages/viewer/src/modules/materials/shaders/speckle-viewport-vert.ts

@@ -153,7 +153,32 @@ void main() {
 		#include <morphnormal_vertex>
 		#include <skinbase_vertex>
 		#include <skinnormal_vertex>
-		#include <defaultnormal_vertex>
+        // #include <defaultnormal_vertex> // COMMENTED CHUNK
+        vec3 transformedNormal = objectNormal;
+        #ifdef USE_INSTANCING
+
+            // this is in lieu of a per-instance normal-matrix
+            // shear transforms in the instance matrix are not supported
+            mat3 m = mat3( instanceMatrix );
+            transformedNormal /= vec3( dot( m[ 0 ], m[ 0 ] ), dot( m[ 1 ], m[ 1 ] ), dot( m[ 2 ], m[ 2 ] ) );
+            transformedNormal = m * transformedNormal;
+            
+            /* If we have negative scaling, we flip the normal */
+            float signDet = sign(dot(m[0], cross(m[1], m[2])));
+            // Optional fallback: treat 0 as +1
+            signDet = signDet + (1.0 - abs(signDet));
+            transformedNormal *= signDet;
+        #endif
+        transformedNormal = normalMatrix * transformedNormal;
+        #ifdef FLIP_SIDED
+            transformedNormal = - transformedNormal;
+        #endif
+        #ifdef USE_TANGENT
+            vec3 transformedTangent = ( modelViewMatrix * vec4( objectTangent, 0.0 ) ).xyz;
+            #ifdef FLIP_SIDED
+                transformedTangent = - transformedTangent;
+            #endif
+        #endif
 	#endif
 	#include <begin_vertex>
 	#include <morphtarget_vertex>

packages/viewer/src/modules/objects/AccelerationStructure.ts

@@ -82,8 +82,8 @@ export class AccelerationStructure {
   }
 
   public static buildBVH(
-    indices: number[] | undefined,
-    position: number[] | undefined,
+    indices: Uint16Array | Uint32Array | undefined,
+    position: Float32Array | Float64Array | undefined,
     options: BVHOptions = DefaultBVHOptions,
     transform?: Matrix4
   ): MeshBVH {
@@ -92,7 +92,7 @@ export class AccelerationStructure {
       throw new Error('Cannot build BVH with undefined indices or position!')
     }
 
-    let bvhPositions = new Float32Array(position)
+    let bvhPositions = position
     if (transform) {
       bvhPositions = new Float32Array(position.length)
       const vecBuff = new Vector3()

packages/viewer/src/modules/objects/TextLabel.ts

@@ -1,6 +1,5 @@
 import {
   Box3,
-  BufferAttribute,
   BufferGeometry,
   Color,
   DoubleSide,
@@ -24,6 +23,7 @@ import SpeckleBasicMaterial, {
 import SpeckleTextMaterial from '../materials/SpeckleTextMaterial.js'
 import { ObjectLayers } from '../../index.js'
 import Logger from '../utils/Logger.js'
+import { Uint16BufferAttribute } from 'three'
 
 const _mat40: Matrix4 = new Matrix4()
 const _mat41: Matrix4 = new Matrix4()
@@ -517,12 +517,9 @@ export class TextLabel extends Text {
     }
 
     const geometry = new BufferGeometry()
-    geometry.setIndex(new BufferAttribute(new Uint32Array(indices), 1))
-    geometry.setAttribute(
-      'position',
-      new BufferAttribute(new Float32Array(positions), 3)
-    )
-    geometry.setAttribute('uv', new BufferAttribute(new Float32Array(uvs), 2))
+    geometry.setIndex(new Uint16BufferAttribute(indices, 1))
+    geometry.setAttribute('position', new Float32BufferAttribute(positions, 3))
+    geometry.setAttribute('uv', new Float32BufferAttribute(uvs, 2))
     geometry.computeBoundingBox()
 
     return geometry

packages/viewer/src/modules/objects/TopLevelAccelerationStructure.ts

@@ -67,24 +67,29 @@ export class TopLevelAccelerationStructure {
   }
 
   private buildBVH() {
-    const indices = []
-    const vertices: number[] = new Array<number>(
+    const indices: Uint16Array = new Uint16Array(
+      TopLevelAccelerationStructure.cubeIndices.length * this.batchObjects.length
+    )
+    const vertices: Float32Array = new Float32Array(
       TopLevelAccelerationStructure.CUBE_VERTS * 3 * this.batchObjects.length
     )
     let vertOffset = 0
+    let indexOffset = 0
     for (let k = 0; k < this.batchObjects.length; k++) {
       const boxBounds: Box3 = this.batchObjects[k].accelerationStructure.getBoundingBox(
         new Box3()
       )
       this.updateVertArray(boxBounds, vertOffset, vertices)
-      indices.push(
-        ...TopLevelAccelerationStructure.cubeIndices.map((val) => val + vertOffset / 3)
+      indices.set(
+        TopLevelAccelerationStructure.cubeIndices.map((val) => val + vertOffset / 3),
+        indexOffset
       )
       this.batchObjects[k].tasVertIndexStart = vertOffset / 3
       this.batchObjects[k].tasVertIndexEnd =
         vertOffset / 3 + TopLevelAccelerationStructure.CUBE_VERTS
 
       vertOffset += TopLevelAccelerationStructure.CUBE_VERTS * 3
+      indexOffset += TopLevelAccelerationStructure.cubeIndices.length
     }
     this.accelerationStructure = new AccelerationStructure(
       AccelerationStructure.buildBVH(indices, vertices)
@@ -105,7 +110,7 @@ export class TopLevelAccelerationStructure {
     }
   }
 
-  private updateVertArray(box: Box3, offset: number, outPositions: number[]) {
+  private updateVertArray(box: Box3, offset: number, outPositions: Float32Array) {
     outPositions[offset] = box.min.x
     outPositions[offset + 1] = box.min.y
     outPositions[offset + 2] = box.max.z
@@ -141,7 +146,7 @@ export class TopLevelAccelerationStructure {
 
   public refit() {
     const positions = this.accelerationStructure.geometry.attributes.position
-      .array as number[]
+      .array as Float32Array
     // const boxBuffer: Box3 = new Box3()
     for (let k = 0; k < this.batchObjects.length; k++) {
       const start = this.batchObjects[k].tasVertIndexStart

packages/viewer/src/modules/tree/NodeRenderView.ts

@@ -1,4 +1,4 @@
-import { Box3 } from 'three'
+import { Box3, Matrix4 } from 'three'
 import { GeometryType } from '../batching/Batch.js'
 import { GeometryAttributes, type GeometryData } from '../converter/Geometry.js'
 import Materials, {
@@ -7,6 +7,7 @@ import Materials, {
   type RenderMaterial
 } from '../materials/Materials.js'
 import { SpeckleType } from '../loaders/GeometryConverter.js'
+import { ChunkArray } from '../converter/VirtualArray.js'
 
 export interface NodeRenderData {
   id: string
@@ -84,6 +85,10 @@ export class NodeRenderView {
     return this._aabb
   }
 
+  public set aabb(value: Box3) {
+    this._aabb.copy(value)
+  }
+
   public get transparent(): boolean {
     return (
       (this._renderData.renderMaterial &&
@@ -150,14 +155,18 @@ export class NodeRenderView {
     if (vertEnd !== undefined) this._batchVertexEnd = vertEnd
   }
 
-  public computeAABB() {
+  public computeAABB(transform?: Matrix4) {
     if (!this._aabb) this._aabb = new Box3()
 
     if (
       this._renderData.geometry.attributes &&
       this._renderData.geometry.attributes.POSITION.length
     ) {
-      this._aabb.setFromArray(this._renderData.geometry.attributes.POSITION)
+      /** For transformations that contain non-uniform scaling combine with rotation the resulting
+       *  aabb is not going to be accurate. We will re-compute and assign it when we build the batches
+       */
+      this._aabb.copy(this._renderData.geometry.attributes.POSITION.computeBox3())
+      if (transform) this._aabb.applyMatrix4(transform)
     }
   }
 
@@ -181,7 +190,9 @@ export class NodeRenderView {
 
   public disposeGeometry() {
     for (const attr in this._renderData.geometry.attributes) {
-      this._renderData.geometry.attributes[attr as GeometryAttributes] = []
+      this._renderData.geometry.attributes[attr as GeometryAttributes] = new ChunkArray(
+        []
+      )
     }
   }
 }

packages/viewer/src/modules/tree/RenderTree.ts

@@ -3,7 +3,6 @@ import { type TreeNode, WorldTree } from './WorldTree.js'
 import Materials from '../materials/Materials.js'
 import { type NodeRenderData, NodeRenderView } from './NodeRenderView.js'
 import { GeometryConverter, SpeckleType } from '../loaders/GeometryConverter.js'
-import { Geometry } from '../converter/Geometry.js'
 import Logger from '../utils/Logger.js'
 
 export class RenderTree {
@@ -51,25 +50,29 @@ export class RenderTree {
   private applyTransforms(node: TreeNode) {
     if (node.model.renderView) {
       const transform = this.computeTransform(node)
-      if (node.model.renderView.hasGeometry) {
+      if (node.model.renderView.hasGeometry || node.model.renderView.hasMetadata) {
         if (node.model.renderView.renderData.geometry.bakeTransform) {
           transform.multiply(node.model.renderView.renderData.geometry.bakeTransform)
         }
-        if (
-          node.model.instanced &&
-          node.model.renderView.speckleType === SpeckleType.Mesh
-        )
-          node.model.renderView.renderData.geometry.transform = transform
-        else {
-          Geometry.transformGeometryData(
-            node.model.renderView.renderData.geometry,
-            transform
-          )
-        }
-        node.model.renderView.computeAABB()
-      } else if (node.model.renderView.hasMetadata) {
-        node.model.renderView.renderData.geometry.bakeTransform.premultiply(transform)
-        node.model.renderView.computeAABB()
+        node.model.renderView.renderData.geometry.transform = transform
+        node.model.renderView.computeAABB(!node.model.instanced ? transform : undefined)
+        /** I like that this is gone now! */
+        //   if (
+        //     node.model.instanced &&
+        //     node.model.renderView.speckleType === SpeckleType.Mesh
+        //   )
+        //     node.model.renderView.renderData.geometry.transform = transform
+        //   else {
+        //     Geometry.transformGeometryData(
+        //       node.model.renderView.renderData.geometry,
+        //       transform
+        //     )
+        //   }
+        //   node.model.renderView.computeAABB()
+        // } else if (node.model.renderView.hasMetadata) {
+        //   node.model.renderView.renderData.geometry.bakeTransform.premultiply(transform)
+        //   node.model.renderView.computeAABB()
+        // }
       }
     }
   }

utils/helm/speckle-server/templates/_helpers.tpl

@@ -1162,6 +1162,12 @@ Generate the environment variables for Speckle server and Speckle objects deploy
 - name: FF_NEXT_GEN_FILE_IMPORTER_ENABLED
   value: {{ .Values.featureFlags.nextGenFileImporterEnabled | quote }}
 {{- end }}
+
+{{- if .Values.featureFlags.rhinoFileImporterEnabled }}
+- name: FF_RHINO_FILE_IMPORTER_ENABLED
+  value: {{ .Values.featureFlags.rhinoFileImporterEnabled  | quote }}
+{{- end }}
+
 {{- if .Values.featureFlags.backgroundJobsEnabled }}
 - name: FILEIMPORT_QUEUE_POSTGRES_URL
   valueFrom:

utils/helm/speckle-server/values.schema.json

@@ -134,6 +134,11 @@
           "type": "boolean",
           "description": "Enables the ability to import data from ACC",
           "default": false
+        },
+        "rhinoFileImporterEnabled": {
+          "type": "boolean",
+          "description": "Enables the dedicated Rhino based file importer. This is not part of the deployment.",
+          "default": false
         }
       }
     },

utils/helm/speckle-server/values.yaml

@@ -75,6 +75,8 @@ featureFlags:
   backgroundJobsEnabled: false
   ## @param featureFlags.accIntegrationEnabled Enables the ability to import data from ACC
   accIntegrationEnabled: false
+  ## @param featureFlags.rhinoFileImporterEnabled Enables the dedicated Rhino based file importer. This is not part of the deployment.
+  rhinoFileImporterEnabled: false
 
 analytics:
   ## @param analytics.enabled Enable or disable analytics