diff --git a/packages/dui3/lib/core/configs/apollo.ts b/packages/dui3/lib/core/configs/apollo.ts index a221f16a6..9582990e2 100644 --- a/packages/dui3/lib/core/configs/apollo.ts +++ b/packages/dui3/lib/core/configs/apollo.ts @@ -4,7 +4,8 @@ import { ApolloLink, InMemoryCache, split, - ApolloClientOptions + ApolloClientOptions, + from } from '@apollo/client/core' import { setContext } from '@apollo/client/link/context' import { SubscriptionClient } from 'subscriptions-transport-ws' @@ -18,6 +19,12 @@ import { buildArrayMergeFunction, incomingOverwritesExistingMergeFunction } from '~~/lib/core/helpers/apolloSetup' +import { onError } from '@apollo/client/link/error' +import { Observability } from '@speckle/shared' + +let subscriptionsStopped = false +const errorRpm = Observability.simpleRpmCounter() +const STOP_SUBSCRIPTIONS_AT_ERRORS_PER_MIN = 100 const appVersion = (import.meta.env.SPECKLE_SERVER_VERSION as string) || 'unknown' const appName = 'dui-3' @@ -289,7 +296,32 @@ function createLink(params: { ) } - return link + const errorLink = onError((res) => { + console.error('Apollo Client error', res) + + // Disable subscriptions if too many errors per minute + const rpm = errorRpm.hit() + if ( + process.client && + wsClient && + !subscriptionsStopped && + rpm > STOP_SUBSCRIPTIONS_AT_ERRORS_PER_MIN + ) { + subscriptionsStopped = true + console.error( + `Too many errors (${rpm} errors per minute), stopping subscriptions!` + ) + wsClient.use([ + { + applyMiddleware: () => { + // never invokes next() - essentially stuck + } + } + ]) + } + }) + + return from([errorLink, link]) } type ResolveClientConfigParams = { diff --git a/packages/frontend-2/components/singleton/AppErrorStateManager.vue b/packages/frontend-2/components/singleton/AppErrorStateManager.vue new file mode 100644 index 000000000..d95256b9a --- /dev/null +++ b/packages/frontend-2/components/singleton/AppErrorStateManager.vue @@ -0,0 +1,33 @@ + + diff --git a/packages/frontend-2/components/singleton/Managers.vue b/packages/frontend-2/components/singleton/Managers.vue index 27dc2158d..a53ca48c9 100644 --- a/packages/frontend-2/components/singleton/Managers.vue +++ b/packages/frontend-2/components/singleton/Managers.vue @@ -1,5 +1,8 @@ diff --git a/packages/frontend/src/main/lib/core/utils/appErrorStateManager.ts b/packages/frontend/src/main/lib/core/utils/appErrorStateManager.ts new file mode 100644 index 000000000..01e4795d7 --- /dev/null +++ b/packages/frontend/src/main/lib/core/utils/appErrorStateManager.ts @@ -0,0 +1,25 @@ +import { Observability } from '@speckle/shared' +import Vue from 'vue' + +const ENTER_STATE_AT_ERRORS_PER_MIN = 100 + +const state = Vue.observable({ + inErrorState: false +}) + +const errorRpm = Observability.simpleRpmCounter() + +export function isErrorState() { + return !!state.inErrorState +} + +export function registerError() { + const epm = errorRpm.hit() + + if (!isErrorState() && epm >= ENTER_STATE_AT_ERRORS_PER_MIN) { + console.error( + `Too many errors (${epm} errors per minute), entering app error state!` + ) + state.inErrorState = true + } +} diff --git a/packages/shared/src/observability/index.ts b/packages/shared/src/observability/index.ts index a2205d1a3..ebdadf23b 100644 --- a/packages/shared/src/observability/index.ts +++ b/packages/shared/src/observability/index.ts @@ -43,3 +43,32 @@ export function extendLoggerComponent( .join('/') return otherChild.child(otherChildBindings) } + +/** + * Very simple RPM counter to catch extreme spam scenarios (e.g. a ton of errors being thrown). It's not going + * to always report accurately, but as long as hits are being registered consistently it should be accurate enough. + */ +export function simpleRpmCounter() { + const getTimestamp = () => new Date().getTime() + let lastDateTimestamp = getTimestamp() + let hits = 0 + + const validateHits = () => { + const timestamp = getTimestamp() + if (timestamp > lastDateTimestamp + 60 * 1000) { + hits = 0 + lastDateTimestamp = timestamp + } + } + + return { + hit: () => { + validateHits() + return ++hits + }, + get: () => { + validateHits() + return hits + } + } +} diff --git a/workspace.code-workspace b/workspace.code-workspace index 99cdc51ff..597043f51 100644 --- a/workspace.code-workspace +++ b/workspace.code-workspace @@ -88,7 +88,7 @@ "volar.vueserver.maxOldSpaceSize": 4000, "cSpell.words": ["Bursty", "mjml"], "tailwindCSS.experimental.configFile": { - "packages/frontend-2/tailwind.config.js": "packages/frontend-2/**" + "packages/frontend-2/tailwind.config.mjs": "packages/frontend-2/**" }, "vue-semantic-server.trace.server": "off", "vue-syntactic-server.trace.server": "off",