From c49a76f87f4d392477fd934719e3ff4cbeeae740 Mon Sep 17 00:00:00 2001
From: Kristaps Fabians Geikins <fabians@speckle.systems>
Date: Fri, 14 Mar 2025 10:43:07 +0200
Subject: [PATCH] feat(fe2): better viewer activity observability (#4190)

---
 .../frontend-2/components/viewer/Base.vue     |  6 +-
 .../lib/viewer/composables/activity.ts        | 37 ++++++++--
 .../lib/viewer/composables/serialization.ts   | 71 ++++++++++---------
 .../lib/viewer/composables/setup/dev.ts       | 37 ++++++++--
 .../frontend-2/type-augmentations/window.d.ts |  6 ++
 5 files changed, 112 insertions(+), 45 deletions(-)

diff --git a/packages/frontend-2/components/viewer/Base.vue b/packages/frontend-2/components/viewer/Base.vue
index ce73ce0dd..9b7f46af0 100644
--- a/packages/frontend-2/components/viewer/Base.vue
+++ b/packages/frontend-2/components/viewer/Base.vue
@@ -1,5 +1,9 @@
 <template>
-  <div ref="rendererparent" class="absolute w-full h-full"></div>
+  <div
+    ref="rendererparent"
+    class="absolute w-full h-full"
+    data-dd-action-name="Viewer Canvas"
+  ></div>
 </template>
 <script setup lang="ts">
 import { useInjectedViewer } from '~~/lib/viewer/composables/setup'
diff --git a/packages/frontend-2/lib/viewer/composables/activity.ts b/packages/frontend-2/lib/viewer/composables/activity.ts
index f8ff0bbd9..0f9ea71ff 100644
--- a/packages/frontend-2/lib/viewer/composables/activity.ts
+++ b/packages/frontend-2/lib/viewer/composables/activity.ts
@@ -13,8 +13,8 @@ import {
   useSelectionEvents,
   useViewerCameraControlEndTracker
 } from '~~/lib/viewer/composables/viewer'
-import { SpeckleViewer } from '@speckle/shared'
-import type { Nullable } from '@speckle/shared'
+import { SpeckleViewer, xor } from '@speckle/shared'
+import type { Nullable, Optional } from '@speckle/shared'
 import { Vector3 } from 'three'
 import { useActiveUser } from '~~/lib/auth/composables/activeUser'
 import { broadcastViewerUserActivityMutation } from '~~/lib/viewer/graphql/mutations'
@@ -82,8 +82,16 @@ export function useViewerUserActivityBroadcasting(
   const apollo = useApolloClient().client
   const { isEnabled: isEmbedEnabled } = useEmbed()
 
+  const isSameState = (
+    a: Optional<ViewerUserActivityMessageInput>,
+    b: Optional<ViewerUserActivityMessageInput>
+  ) => {
+    if (xor(a, b)) return false
+    if (!a || !b) return false
+    return JSON.stringify(a) === JSON.stringify(b)
+  }
+
   const invokeMutation = async (message: ViewerUserActivityMessageInput) => {
-    if (!isLoggedIn.value || isEmbedEnabled.value) return false
     const result = await apollo
       .mutate({
         mutation: broadcastViewerUserActivityMutation,
@@ -98,14 +106,33 @@ export function useViewerUserActivityBroadcasting(
     return result.data?.broadcastViewerUserActivity || false
   }
 
+  let previousMessage: Optional<ViewerUserActivityMessageInput> = undefined
+  const invokeObservabilityEvent = async (message: ViewerUserActivityMessageInput) => {
+    const dd = window.DD_RUM
+    if (!dd || !('addAction' in dd)) return
+
+    if (isSameState(previousMessage, message)) return
+
+    previousMessage = message
+    dd.addAction('Viewer User Activity', { message })
+  }
+
+  const invoke = async (message: ViewerUserActivityMessageInput) => {
+    if (!isLoggedIn.value || isEmbedEnabled.value) return false
+    return await Promise.all([
+      invokeMutation(message),
+      invokeObservabilityEvent(message)
+    ])
+  }
+
   return {
     emitDisconnected: async () =>
-      invokeMutation({
+      await invoke({
         ...getMainMetadata(),
         status: ViewerUserActivityStatus.Disconnected
       }),
     emitViewing: async () => {
-      await invokeMutation({
+      await invoke({
         ...getMainMetadata(),
         status: ViewerUserActivityStatus.Viewing
       })
diff --git a/packages/frontend-2/lib/viewer/composables/serialization.ts b/packages/frontend-2/lib/viewer/composables/serialization.ts
index 5b240f8a1..920061098 100644
--- a/packages/frontend-2/lib/viewer/composables/serialization.ts
+++ b/packages/frontend-2/lib/viewer/composables/serialization.ts
@@ -10,8 +10,9 @@ import {
   useFilterUtilities,
   useSelectionUtilities
 } from '~~/lib/viewer/composables/ui'
-import { CameraController, ViewMode } from '@speckle/viewer'
+import { CameraController, ViewMode, VisualDiffMode } from '@speckle/viewer'
 import type { NumericPropertyInfo } from '@speckle/viewer'
+import type { PartialDeep } from 'type-fest'
 
 type SerializedViewerState = SpeckleViewer.ViewerState.SerializedViewerState
 
@@ -165,61 +166,61 @@ export function useApplySerializedState() {
   const { setSelectionFromObjectIds } = useSelectionUtilities()
   const logger = useLogger()
 
-  return async (state: SerializedViewerState, mode: StateApplyMode) => {
+  return async (state: PartialDeep<SerializedViewerState>, mode: StateApplyMode) => {
     if (mode === StateApplyMode.Reset) {
       resetState()
       return
     }
 
     position.value = new Vector3(
-      state.ui.camera.position[0],
-      state.ui.camera.position[1],
-      state.ui.camera.position[2]
+      state.ui?.camera?.position?.[0],
+      state.ui?.camera?.position?.[1],
+      state.ui?.camera?.position?.[2]
     )
     target.value = new Vector3(
-      state.ui.camera.target[0],
-      state.ui.camera.target[1],
-      state.ui.camera.target[2]
+      state.ui?.camera?.target?.[0],
+      state.ui?.camera?.target?.[1],
+      state.ui?.camera?.target?.[2]
     )
 
-    isOrthoProjection.value = state.ui.camera.isOrthoProjection
+    isOrthoProjection.value = !!state.ui?.camera?.isOrthoProjection
 
-    sectionBox.value = state.ui.sectionBox
+    sectionBox.value = state.ui?.sectionBox
       ? new Box3(
           new Vector3(
-            state.ui.sectionBox.min[0],
-            state.ui.sectionBox.min[1],
-            state.ui.sectionBox.min[2]
+            state.ui.sectionBox.min?.[0],
+            state.ui.sectionBox.min?.[1],
+            state.ui.sectionBox.min?.[2]
           ),
           new Vector3(
-            state.ui.sectionBox.max[0],
-            state.ui.sectionBox.max[1],
-            state.ui.sectionBox.max[2]
+            state.ui.sectionBox.max?.[0],
+            state.ui.sectionBox.max?.[1],
+            state.ui.sectionBox.max?.[2]
           )
         )
       : null
 
-    const filters = state.ui.filters
-    if (filters.hiddenObjectIds.length) {
+    const filters = state.ui?.filters || {}
+    if (filters.hiddenObjectIds?.length) {
       resetFilters()
       hideObjects(filters.hiddenObjectIds, { replace: true })
-    } else if (filters.isolatedObjectIds.length) {
+    } else if (filters.isolatedObjectIds?.length) {
       resetFilters()
       isolateObjects(filters.isolatedObjectIds, { replace: true })
     } else {
       resetFilters()
     }
 
-    const propertyFilterApplied = state.ui.filters.propertyFilter.isApplied
+    const propertyFilterApplied = filters.propertyFilter?.isApplied
     if (propertyFilterApplied) {
       applyPropertyFilter()
     } else {
       unApplyPropertyFilter()
     }
 
-    const propertyInfoKey = state.ui.filters.propertyFilter.key
-    const passMin = state.viewer.metadata.filteringState?.passMin
-    const passMax = state.viewer.metadata.filteringState?.passMax
+    const propertyInfoKey = filters.propertyFilter?.key
+    const passMin = state.viewer?.metadata?.filteringState?.passMin
+    const passMax = state.viewer?.metadata?.filteringState?.passMax
     if (propertyInfoKey) {
       removePropertyFilter()
 
@@ -249,9 +250,9 @@ export function useApplySerializedState() {
     }
 
     if (mode === StateApplyMode.Spotlight) {
-      highlightedObjectIds.value = filters.selectedObjectIds.slice()
+      highlightedObjectIds.value = (filters.selectedObjectIds || []).slice()
     } else {
-      if (filters.selectedObjectIds.length) {
+      if (filters.selectedObjectIds?.length) {
         setSelectionFromObjectIds(filters.selectedObjectIds)
       }
     }
@@ -259,20 +260,22 @@ export function useApplySerializedState() {
     if (
       [StateApplyMode.Spotlight, StateApplyMode.TheadFullContextOpen].includes(mode)
     ) {
-      await resourceIdString.update(state.resources.request.resourceIdString)
+      await resourceIdString.update(state.resources?.request?.resourceIdString || '')
     }
 
     if ([StateApplyMode.Spotlight].includes(mode)) {
-      await urlHashState.focusedThreadId.update(state.ui.threads.openThread.threadId)
+      await urlHashState.focusedThreadId.update(
+        state.ui?.threads?.openThread?.threadId || null
+      )
     }
 
-    const command = state.ui.diff.command
+    const command = state.ui?.diff?.command
       ? deserializeDiffCommand(state.ui.diff.command)
       : null
     const activeDiffEnabled = !!diff.enabled.value
-    if (command && command.diffs.length) {
-      diff.time.value = state.ui.diff.time
-      diff.mode.value = state.ui.diff.mode
+    if (command && command.diffs.length && state.ui?.diff) {
+      diff.time.value = state.ui.diff.time || 0.5
+      diff.mode.value = state.ui?.diff.mode || VisualDiffMode.COLORED
 
       const instruction = command.diffs[0]
       await diffModelVersions(
@@ -285,16 +288,16 @@ export function useApplySerializedState() {
     }
 
     // Restore view mode
-    if (state.ui.viewMode) {
+    if (state.ui?.viewMode) {
       viewMode.value = state.ui.viewMode
     } else {
       viewMode.value = ViewMode.DEFAULT
     }
 
-    explodeFactor.value = state.ui.explodeFactor
+    explodeFactor.value = state.ui?.explodeFactor || 0
     lightConfig.value = {
       ...lightConfig.value,
-      ...state.ui.lightConfig
+      ...(state.ui?.lightConfig || {})
     }
   }
 }
diff --git a/packages/frontend-2/lib/viewer/composables/setup/dev.ts b/packages/frontend-2/lib/viewer/composables/setup/dev.ts
index 791cb1d66..7d54c3292 100644
--- a/packages/frontend-2/lib/viewer/composables/setup/dev.ts
+++ b/packages/frontend-2/lib/viewer/composables/setup/dev.ts
@@ -1,6 +1,12 @@
 import { ViewerEvent } from '@speckle/viewer'
+import {
+  StateApplyMode,
+  useApplySerializedState
+} from '~/lib/viewer/composables/serialization'
 import { useInjectedViewerState } from '~~/lib/viewer/composables/setup'
 import { useViewerEventListener } from '~~/lib/viewer/composables/viewer'
+import type { SpeckleViewer } from '@speckle/shared'
+import { get } from 'lodash-es'
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 function useDebugViewerEvents() {
@@ -13,21 +19,42 @@ function useDebugViewerEvents() {
 
 function useDebugViewer() {
   const state = useInjectedViewerState()
+  const apply = useApplySerializedState()
   const {
     viewer: { instance }
   } = state
 
-  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-  // @ts-expect-error
+  // Get current viewer instance
   window.VIEWER = instance
-  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-  // @ts-expect-error
+
+  // Get current viewer state
   window.VIEWER_STATE = () => state
+
+  // Apply viewer state
+  window.APPLY_VIEWER_STATE = (
+    state: SpeckleViewer.ViewerState.SerializedViewerState
+  ) => apply(state, StateApplyMode.TheadFullContextOpen)
+
+  // Apply DD user activity event
+  window.APPLY_VIEWER_DD_EVENT = (event: {
+    content: {
+      attributes: {
+        context: { message: { state: SpeckleViewer.ViewerState.SerializedViewerState } }
+      }
+    }
+  }) => {
+    const path = 'content.attributes.context.message.state'
+    const state = get(event, path)
+    if (!state) {
+      throw new Error('Cant find serialized state at path: ' + path)
+    }
+
+    return apply(state, StateApplyMode.TheadFullContextOpen)
+  }
 }
 
 export function setupDebugMode() {
   if (import.meta.server) return
-  if (!import.meta.dev) return
 
   // useDebugViewerEvents()
   useDebugViewer()
diff --git a/packages/frontend-2/type-augmentations/window.d.ts b/packages/frontend-2/type-augmentations/window.d.ts
index b35f7a450..7065b2647 100644
--- a/packages/frontend-2/type-augmentations/window.d.ts
+++ b/packages/frontend-2/type-augmentations/window.d.ts
@@ -7,6 +7,12 @@ declare global {
      * Start a new DD RUM view. Function is idempotent and can be safely called multiple times.
      */
     DD_RUM_START_VIEW?: (path: string, name: string) => void
+
+    // Debug keys, don't need to type properly cause we only use them manually from dev tools
+    VIEWER?: any
+    VIEWER_STATE?: any
+    APPLY_VIEWER_STATE?: any
+    APPLY_VIEWER_DD_EVENT?: any
   }
 }