feat: stopping subscriptions at >100 errors per minute (#1592)

* feat: stopping subscriptions at >100 errors per minute

* chore: added explanatory comments

* added error state banner in fe1

* feat: error state banner in fe2

packages/dui3/lib/core/configs/apollo.ts

@@ -4,7 +4,8 @@ import {
   ApolloLink,
   InMemoryCache,
   split,
-  ApolloClientOptions
+  ApolloClientOptions,
+  from
 } from '@apollo/client/core'
 import { setContext } from '@apollo/client/link/context'
 import { SubscriptionClient } from 'subscriptions-transport-ws'
@@ -18,6 +19,12 @@ import {
   buildArrayMergeFunction,
   incomingOverwritesExistingMergeFunction
 } from '~~/lib/core/helpers/apolloSetup'
+import { onError } from '@apollo/client/link/error'
+import { Observability } from '@speckle/shared'
+
+let subscriptionsStopped = false
+const errorRpm = Observability.simpleRpmCounter()
+const STOP_SUBSCRIPTIONS_AT_ERRORS_PER_MIN = 100
 
 const appVersion = (import.meta.env.SPECKLE_SERVER_VERSION as string) || 'unknown'
 const appName = 'dui-3'
@@ -289,7 +296,32 @@ function createLink(params: {
     )
   }
 
-  return link
+  const errorLink = onError((res) => {
+    console.error('Apollo Client error', res)
+
+    // Disable subscriptions if too many errors per minute
+    const rpm = errorRpm.hit()
+    if (
+      process.client &&
+      wsClient &&
+      !subscriptionsStopped &&
+      rpm > STOP_SUBSCRIPTIONS_AT_ERRORS_PER_MIN
+    ) {
+      subscriptionsStopped = true
+      console.error(
+        `Too many errors (${rpm} errors per minute), stopping subscriptions!`
+      )
+      wsClient.use([
+        {
+          applyMiddleware: () => {
+            // never invokes next() - essentially stuck
+          }
+        }
+      ])
+    }
+  })
+
+  return from([errorLink, link])
 }
 
 type ResolveClientConfigParams = {

packages/frontend-2/components/singleton/AppErrorStateManager.vue

@@ -0,0 +1,33 @@
+<template>
+  <div>
+    <div
+      v-if="isErrorState && showBanner"
+      class="fixed inset-x-0 top-0 p-2 z-[1000] bg-danger text-foundation flex justify-between items-center"
+    >
+      <div>
+        Due to a large amount of errors some functionality has been disabled! Please
+        reload the page or contact the server administrators.
+      </div>
+      <div>
+        <FormButton
+          hide-text
+          :icon-left="XMarkIcon"
+          color="invert"
+          text
+          size="lg"
+          @click="hideErrorStateBanner"
+        />
+      </div>
+    </div>
+  </div>
+</template>
+<script setup lang="ts">
+import { XMarkIcon } from '@heroicons/vue/24/solid'
+import { useAppErrorState } from '~~/lib/core/composables/appErrorState'
+
+const { isErrorState } = useAppErrorState()
+
+const showBanner = ref(true)
+
+const hideErrorStateBanner = () => (showBanner.value = false)
+</script>

packages/frontend-2/components/singleton/Managers.vue

@@ -1,5 +1,8 @@
 <template>
-  <SingletonToastManager />
+  <div>
+    <SingletonToastManager />
+    <SingletonAppErrorStateManager />
+  </div>
 </template>
 <script setup lang="ts">
 // This just wraps all global singleton/manager components that should be always available in all layouts

packages/frontend-2/lib/core/composables/appErrorState.ts

@@ -0,0 +1,25 @@
+import { useScopedState } from '~~/lib/common/composables/scopedState'
+import { Observability } from '@speckle/shared'
+
+const ENTER_STATE_AT_ERRORS_PER_MIN = 100
+
+export function useAppErrorState() {
+  const state = useScopedState('appErrorState', () => ({
+    inErrorState: ref(false),
+    errorRpm: Observability.simpleRpmCounter()
+  }))
+
+  return {
+    isErrorState: computed(() => state.inErrorState.value),
+    registerError: () => {
+      const epm = state.errorRpm.hit()
+
+      if (!state.inErrorState.value && epm >= ENTER_STATE_AT_ERRORS_PER_MIN) {
+        console.error(
+          `Too many errors (${epm} errors per minute), entering app error state!`
+        )
+        state.inErrorState.value = true
+      }
+    }
+  }
+}

packages/frontend-2/lib/core/configs/apollo.ts

@@ -24,6 +24,7 @@ import {
 } from '~~/lib/core/helpers/apolloSetup'
 import { onError } from '@apollo/client/link/error'
 import { useNavigateToLogin } from '~~/lib/common/helpers/route'
+import { useAppErrorState } from '~~/lib/core/composables/appErrorState'
 
 const appVersion = (import.meta.env.SPECKLE_SERVER_VERSION as string) || 'unknown'
 const appName = 'frontend-2'
@@ -273,8 +274,11 @@ function createLink(params: {
 }): ApolloLink {
   const { httpEndpoint, wsClient, authToken } = params
   const goToLogin = useNavigateToLogin()
+  const { registerError, isErrorState } = useAppErrorState()
 
   const errorLink = onError((res) => {
+    console.error('Apollo Client error', res)
+
     const { networkError } = res
     if (networkError && isServerError(networkError)) {
       const isForbidden = networkError.statusCode === 403
@@ -284,6 +288,8 @@ function createLink(params: {
         goToLogin()
       }
     }
+
+    registerError()
   })
 
   // Prepare links
@@ -319,6 +325,19 @@ function createLink(params: {
       wsLink,
       link
     )
+
+    // Stopping WS when in error state
+    wsClient.use([
+      {
+        applyMiddleware: (_opt, next) => {
+          if (isErrorState.value) {
+            return // never invokes next() - essentially stuck
+          }
+
+          next()
+        }
+      }
+    ])
   }
 
   return from([errorLink, link])

packages/frontend/src/config/apolloConfig.ts

@@ -5,7 +5,8 @@ import {
   ApolloLink,
   InMemoryCache,
   split,
-  TypePolicies
+  TypePolicies,
+  from
 } from '@apollo/client/core'
 import { setContext } from '@apollo/client/link/context'
 import { WebSocketLink } from '@apollo/client/link/ws'
@@ -22,6 +23,8 @@ import {
 import { merge } from 'lodash'
 import { statePolicies as commitObjectViewerStatePolicies } from '@/main/lib/viewer/commit-object-viewer/stateManagerCore'
 import { Optional } from '@speckle/shared'
+import { onError } from '@apollo/client/link/error'
+import { registerError, isErrorState } from '@/main/lib/core/utils/appErrorStateManager'
 
 // Name of the localStorage item
 const AUTH_TOKEN = LocalStorageKeys.AuthToken
@@ -197,6 +200,7 @@ function createLink(wsClient?: SubscriptionClient): ApolloLink {
   })
   let link = authLink.concat(httpLink)
 
+  // WS link
   if (wsClient) {
     const wsLink = new WebSocketLink(wsClient)
     link = split(
@@ -209,9 +213,27 @@ function createLink(wsClient?: SubscriptionClient): ApolloLink {
       wsLink,
       link
     )
+
+    // Stopping WS when in error state
+    wsClient.use([
+      {
+        applyMiddleware: (_opt, next) => {
+          if (isErrorState()) {
+            return // never invokes next() - essentially stuck
+          }
+
+          next()
+        }
+      }
+    ])
   }
 
-  return link
+  // Global error handling
+  const errorLink = onError(() => {
+    registerError()
+  })
+
+  return from([errorLink, link])
 }
 
 function createApolloClient() {

packages/frontend/src/main/App.vue

@@ -1,11 +1,53 @@
 <template>
-  <router-view></router-view>
+  <div>
+    <div v-if="isAppErrorState && showBanner" class="app-error-state">
+      <div class="app-error-state__wrapper">
+        <div>
+          Due to a large amount of errors some functionality has been disabled! Please
+          reload the page or contact the server administrators.
+        </div>
+        <div>
+          <v-btn v-tooltip="'Close banner'" icon @click="hideErrorStateBanner">
+            <v-icon class="app-error-state__icon">mdi-close-circle</v-icon>
+          </v-btn>
+        </div>
+      </div>
+    </div>
+    <router-view></router-view>
+  </div>
 </template>
-<script>
-export default {}
+<script setup lang="ts">
+import { isErrorState } from '@/main/lib/core/utils/appErrorStateManager'
+import { computed, ref } from 'vue'
+
+const showBanner = ref(true)
+const isAppErrorState = computed(() => isErrorState())
+const hideErrorStateBanner = () => (showBanner.value = false)
 </script>
 <style lang="css">
 .v-timeline:before {
   top: 40px !important;
 }
+
+.app-error-state {
+  position: fixed;
+  left: 0;
+  right: 0;
+  top: 0;
+  background-color: red;
+  z-index: 1000;
+  padding: 8px;
+  font-family: 'Roboto', sans-serif !important;
+  color: white;
+}
+
+.app-error-state__wrapper {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+}
+
+.app-error-state__icon {
+  color: white !important;
+}
 </style>

packages/frontend/src/main/lib/core/utils/appErrorStateManager.ts

@@ -0,0 +1,25 @@
+import { Observability } from '@speckle/shared'
+import Vue from 'vue'
+
+const ENTER_STATE_AT_ERRORS_PER_MIN = 100
+
+const state = Vue.observable({
+  inErrorState: false
+})
+
+const errorRpm = Observability.simpleRpmCounter()
+
+export function isErrorState() {
+  return !!state.inErrorState
+}
+
+export function registerError() {
+  const epm = errorRpm.hit()
+
+  if (!isErrorState() && epm >= ENTER_STATE_AT_ERRORS_PER_MIN) {
+    console.error(
+      `Too many errors (${epm} errors per minute), entering app error state!`
+    )
+    state.inErrorState = true
+  }
+}

packages/shared/src/observability/index.ts

@@ -43,3 +43,32 @@ export function extendLoggerComponent(
     .join('/')
   return otherChild.child(otherChildBindings)
 }
+
+/**
+ * Very simple RPM counter to catch extreme spam scenarios (e.g. a ton of errors being thrown). It's not going
+ * to always report accurately, but as long as hits are being registered consistently it should be accurate enough.
+ */
+export function simpleRpmCounter() {
+  const getTimestamp = () => new Date().getTime()
+  let lastDateTimestamp = getTimestamp()
+  let hits = 0
+
+  const validateHits = () => {
+    const timestamp = getTimestamp()
+    if (timestamp > lastDateTimestamp + 60 * 1000) {
+      hits = 0
+      lastDateTimestamp = timestamp
+    }
+  }
+
+  return {
+    hit: () => {
+      validateHits()
+      return ++hits
+    },
+    get: () => {
+      validateHits()
+      return hits
+    }
+  }
+}

workspace.code-workspace

@@ -88,7 +88,7 @@
     "volar.vueserver.maxOldSpaceSize": 4000,
     "cSpell.words": ["Bursty", "mjml"],
     "tailwindCSS.experimental.configFile": {
-      "packages/frontend-2/tailwind.config.js": "packages/frontend-2/**"
+      "packages/frontend-2/tailwind.config.mjs": "packages/frontend-2/**"
     },
     "vue-semantic-server.trace.server": "off",
     "vue-syntactic-server.trace.server": "off",