Iain Sproat
8197bb74a3
* feat(multi-region): metrics for knex for all regional databases * improve typing in knex monitoring * error logging around migrations * await async calls for db connections - add 'region' label * add missing 'await' statements * more missing 'await' * guard against re-adding listeners * It was possible for update to be called before initialize - this change collapses both into initialize, and adds checks to ensure initialization is done before being updated for new regions * separate back into non-exported const and rename * align with main * Amend order at which metrics is enabled
82 lines
2.3 KiB
TypeScript
82 lines
2.3 KiB
TypeScript
/**
|
|
* High frequency monitoring, collects data related to CPU, memory, database, and network usage
|
|
* at a higher frequency than the default prometheus monitoring. It makes the data
|
|
* available to Prometheus via an histogram.
|
|
*/
|
|
|
|
import { Histogram, Registry } from 'prom-client'
|
|
import { processCpuTotal } from '@/logging/highFrequencyMetrics/processCPUTotal'
|
|
import { heapSizeAndUsed } from '@/logging/highFrequencyMetrics/heapSizeAndUsed'
|
|
import { knexConnections } from '@/logging/highFrequencyMetrics/knexConnectionPool'
|
|
import { type Knex } from 'knex'
|
|
|
|
type MetricConfig = {
|
|
prefix?: string
|
|
labels?: Record<string, string>
|
|
buckets?: Record<string, number[]>
|
|
getDbClients: () => Promise<
|
|
Array<{ client: Knex; isMain: boolean; regionKey: string }>
|
|
>
|
|
}
|
|
|
|
type HighFrequencyMonitor = {
|
|
start: () => () => void
|
|
}
|
|
|
|
export const initHighFrequencyMonitoring = (params: {
|
|
register: Registry
|
|
collectionPeriodMilliseconds: number
|
|
config: MetricConfig
|
|
}): HighFrequencyMonitor => {
|
|
const { register, collectionPeriodMilliseconds } = params
|
|
const config = params.config
|
|
const registers = register ? [register] : undefined
|
|
const namePrefix = config.prefix ?? ''
|
|
const labels = config.labels ?? {}
|
|
const labelNames = Object.keys(labels)
|
|
|
|
const metrics = [
|
|
processCpuTotal(register, config),
|
|
heapSizeAndUsed(register, config),
|
|
knexConnections(register, config)
|
|
]
|
|
|
|
const selfMonitor = new Histogram({
|
|
name: namePrefix + 'self_monitor_time_high_frequency',
|
|
help: 'The time taken to collect all of the high frequency metrics, seconds.',
|
|
registers,
|
|
buckets: [0, 0.001, 0.01, 0.025, 0.05, 0.1, 0.2],
|
|
labelNames
|
|
})
|
|
|
|
return {
|
|
start: collectHighFrequencyMetrics({
|
|
selfMonitor,
|
|
metrics,
|
|
collectionPeriodMilliseconds
|
|
})
|
|
}
|
|
}
|
|
|
|
export interface Metric {
|
|
collect: () => void
|
|
}
|
|
|
|
const collectHighFrequencyMetrics = (params: {
|
|
selfMonitor: Histogram<string>
|
|
collectionPeriodMilliseconds: number
|
|
metrics: Metric[]
|
|
}) => {
|
|
const { selfMonitor, metrics, collectionPeriodMilliseconds } = params
|
|
return () => {
|
|
const intervalId = setInterval(() => {
|
|
const end = selfMonitor.startTimer()
|
|
for (const metric of metrics) {
|
|
metric.collect()
|
|
}
|
|
end()
|
|
}, collectionPeriodMilliseconds)
|
|
return () => clearInterval(intervalId)
|
|
}
|
|
}
|