speckle-server/packages/server/modules/previews/observability/metrics.ts

import { TIME } from '@speckle/shared'
import Bull from 'bull'
import { type Registry, Counter, Summary, Gauge } from 'prom-client'
import { ObserveMetrics } from '@/modules/previews/domain/operations'

export const PreviewJobDurationStep = {
  TOTAL: 'total',
  LOAD: 'load',
  RENDER: 'render'
} as const

export const initializeMetrics = (params: {
  registers: Registry[]
  previewRequestQueue: Bull.Queue
  previewResponseQueue: Bull.Queue
}) => {
  const { registers, previewRequestQueue, previewResponseQueue } = params

  // ======= Request Queue =======
  // add a metric to gauge the length of the preview job queue
  registers.forEach((r) =>
    r.removeSingleMetric('speckle_server_preview_jobs_request_queue_pending')
  )
  new Gauge({
    name: 'speckle_server_preview_jobs_request_queue_pending',
    help: 'Number of preview jobs waiting in the job request queue',
    async collect() {
      this.set(await previewRequestQueue.count())
    }
  })

  registers.forEach((r) =>
    r.removeSingleMetric('speckle_server_preview_jobs_request_waiting_count')
  )
  const previewJobsRequestWaitingCounter = new Counter({
    name: 'speckle_server_preview_jobs_request_waiting_count',
    help: 'Total number of preview jobs which have been added to the queue to be processed (and are in a waiting state).'
  })

  registers.forEach((r) =>
    r.removeSingleMetric('speckle_server_preview_jobs_request_active_count')
  )
  const previewJobsRequestActiveCounter = new Counter({
    name: 'speckle_server_preview_jobs_request_active_count',
    help: 'Total number of preview jobs which have been requested and were being processed (are in an active state).'
  })

  registers.forEach((r) =>
    r.removeSingleMetric('speckle_server_preview_jobs_request_completed_count')
  )
  const previewJobsRequestCompletedCounter = new Counter({
    name: 'speckle_server_preview_jobs_request_completed_count',
    help: 'Total number of preview jobs which have been requested and were successfully completed by a worker.'
  })

  registers.forEach((r) =>
    r.removeSingleMetric('speckle_server_preview_jobs_request_failed_count')
  )
  const previewJobsRequestFailedCounter = new Counter({
    name: 'speckle_server_preview_jobs_request_failed_count',
    help: 'Total number of preview jobs which have been requested and were not successful (failed).'
  })

  const waitingHandler = () => {
    previewJobsRequestWaitingCounter.inc()
  }
  previewRequestQueue.removeListener('waiting', waitingHandler)
  previewRequestQueue.on('waiting', waitingHandler)

  const completedHandler = () => {
    previewJobsRequestCompletedCounter.inc()
  }
  previewRequestQueue.removeListener('completed', completedHandler)
  previewRequestQueue.on('completed', completedHandler)

  const activeHandler = () => {
    previewJobsRequestActiveCounter.inc()
  }
  previewRequestQueue.removeListener('active', activeHandler)
  previewRequestQueue.on('active', activeHandler)

  const failedHandler = () => {
    previewJobsRequestFailedCounter.inc()
  }
  previewRequestQueue.removeListener('failed', failedHandler)
  previewRequestQueue.on('failed', failedHandler)

  // ======= Response Queue =======
  registers.forEach((r) =>
    r.removeSingleMetric('speckle_server_preview_jobs_response_queue_pending')
  )
  new Gauge({
    name: 'speckle_server_preview_jobs_response_queue_pending',
    help: 'Number of responses to preview jobs waiting in the response queue',
    async collect() {
      this.set(await previewResponseQueue.count())
    }
  })

  registers.forEach((r) =>
    r.removeSingleMetric('speckle_server_preview_jobs_response_completed_count')
  )
  const previewJobsResponseCompletedCounter = new Counter({
    name: 'speckle_server_preview_jobs_response_completed_count',
    help: 'Total number of preview jobs which have been responded and the response has been successfully processed.'
  })

  registers.forEach((r) =>
    r.removeSingleMetric('speckle_server_preview_jobs_response_failed_count')
  )
  const previewJobsResponseFailedCounter = new Counter({
    name: 'speckle_server_preview_jobs_response_failed_count',
    help: 'Total number of preview jobs which have been responded and the response has not been successfully processed.'
  })

  const responseCompletedHandler = () => {
    previewJobsResponseCompletedCounter.inc()
  }
  previewResponseQueue.removeListener('completed', responseCompletedHandler)
  previewResponseQueue.on('completed', responseCompletedHandler)

  const responseFailedHandler = () => {
    previewJobsResponseFailedCounter.inc()
  }
  previewResponseQueue.removeListener('failed', responseFailedHandler)
  previewResponseQueue.on('failed', responseFailedHandler)

  // ======= Responses =======

  registers.forEach((r) =>
    r.removeSingleMetric('speckle_server_preview_jobs_processed_duration_seconds')
  )
  const previewJobsProcessedSummary = new Summary<'status' | 'step'>({
    name: 'speckle_server_preview_jobs_processed_duration_seconds',
    help: 'Duration of preview job processing, in seconds, as sampled over a moving window of 1 minute.',
    labelNames: ['status', 'step'],
    maxAgeSeconds: 1 * TIME.minute,
    ageBuckets: 5
  })

  return { previewJobsProcessedSummary }
}

export const observeMetricsFactory = (deps: {
  summary: Summary<'step' | 'status'>
}): ObserveMetrics => {
  const { summary } = deps
  return (params) => {
    const { payload } = params
    summary.observe(
      { status: payload.status, step: PreviewJobDurationStep.TOTAL },
      payload.result.durationSeconds * TIME.second
    )
    if (payload.result.loadDurationSeconds) {
      summary.observe(
        { status: payload.status, step: PreviewJobDurationStep.LOAD },
        payload.result.loadDurationSeconds * TIME.second
      )
    }
    if (payload.result.renderDurationSeconds) {
      summary.observe(
        { status: payload.status, step: PreviewJobDurationStep.RENDER },
        payload.result.renderDurationSeconds * TIME.second
      )
    }
  }
}