From 8c5d83eb4f2cf00c9b623ddd271b7b43a3b74dc2 Mon Sep 17 00:00:00 2001 From: Alessandro Magionami Date: Fri, 25 Oct 2024 15:55:49 +0200 Subject: [PATCH] chore(fileimport): parser and prometheurMetrics refactor multiregion --- packages/fileimport-service/ifc/parser.js | 4 +- .../src/prometheusMetrics.js | 156 +++++++++--------- 2 files changed, 81 insertions(+), 79 deletions(-) diff --git a/packages/fileimport-service/ifc/parser.js b/packages/fileimport-service/ifc/parser.js index c6d33270c..021760a72 100644 --- a/packages/fileimport-service/ifc/parser.js +++ b/packages/fileimport-service/ifc/parser.js @@ -4,9 +4,9 @@ const { logger } = require('../observability/logging.js') const ServerAPI = require('./api.js') module.exports = class IFCParser { - constructor({ serverApi, logger }) { + constructor({ db, serverApi, logger }) { this.api = new WebIFC.IfcAPI() - this.serverApi = serverApi || new ServerAPI({ logger }) + this.serverApi = serverApi || new ServerAPI({ db, logger }) } async parse(data) { diff --git a/packages/fileimport-service/src/prometheusMetrics.js b/packages/fileimport-service/src/prometheusMetrics.js index aadae3879..acf84fe47 100644 --- a/packages/fileimport-service/src/prometheusMetrics.js +++ b/packages/fileimport-service/src/prometheusMetrics.js @@ -24,101 +24,103 @@ prometheusClient.collectDefaultMetrics() let prometheusInitialized = false -function initKnexPrometheusMetrics() { - metricFree = new prometheusClient.Gauge({ - name: 'speckle_server_knex_free', - help: 'Number of free DB connections', - collect() { - this.set(knex.client.pool.numFree()) - } - }) +const initDBPrometheusMetricsFactory = + ({ db }) => + () => { + metricFree = new prometheusClient.Gauge({ + name: 'speckle_server_knex_free', + help: 'Number of free DB connections', + collect() { + this.set(db.client.pool.numFree()) + } + }) - metricUsed = new prometheusClient.Gauge({ - name: 'speckle_server_knex_used', - help: 'Number of used DB connections', - collect() { - this.set(knex.client.pool.numUsed()) - } - }) + metricUsed = new prometheusClient.Gauge({ + name: 'speckle_server_knex_used', + help: 'Number of used DB connections', + collect() { + this.set(db.client.pool.numUsed()) + } + }) - metricPendingAquires = new prometheusClient.Gauge({ - name: 'speckle_server_knex_pending', - help: 'Number of pending DB connection aquires', - collect() { - this.set(knex.client.pool.numPendingAcquires()) - } - }) + metricPendingAquires = new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending', + help: 'Number of pending DB connection aquires', + collect() { + this.set(db.client.pool.numPendingAcquires()) + } + }) - metricPendingCreates = new prometheusClient.Gauge({ - name: 'speckle_server_knex_pending_creates', - help: 'Number of pending DB connection creates', - collect() { - this.set(knex.client.pool.numPendingCreates()) - } - }) + metricPendingCreates = new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending_creates', + help: 'Number of pending DB connection creates', + collect() { + this.set(db.client.pool.numPendingCreates()) + } + }) - metricPendingValidations = new prometheusClient.Gauge({ - name: 'speckle_server_knex_pending_validations', - help: 'Number of pending DB connection validations. This is a state between pending acquisition and acquiring a connection.', - collect() { - this.set(knex.client.pool.numPendingValidations()) - } - }) + metricPendingValidations = new prometheusClient.Gauge({ + name: 'speckle_server_knex_pending_validations', + help: 'Number of pending DB connection validations. This is a state between pending acquisition and acquiring a connection.', + collect() { + this.set(db.client.pool.numPendingValidations()) + } + }) - metricRemainingCapacity = new prometheusClient.Gauge({ - name: 'speckle_server_knex_remaining_capacity', - help: 'Remaining capacity of the DB connection pool', - collect() { - const postgresMaxConnections = - parseInt(process.env.POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE) || 1 - const demand = - knex.client.pool.numUsed() + - knex.client.pool.numPendingCreates() + - knex.client.pool.numPendingValidations() + - knex.client.pool.numPendingAcquires() + metricRemainingCapacity = new prometheusClient.Gauge({ + name: 'speckle_server_knex_remaining_capacity', + help: 'Remaining capacity of the DB connection pool', + collect() { + const postgresMaxConnections = + parseInt(process.env.POSTGRES_MAX_CONNECTIONS_FILE_IMPORT_SERVICE) || 1 + const demand = + db.client.pool.numUsed() + + db.client.pool.numPendingCreates() + + db.client.pool.numPendingValidations() + + db.client.pool.numPendingAcquires() - this.set(Math.max(postgresMaxConnections - demand, 0)) - } - }) + this.set(Math.max(postgresMaxConnections - demand, 0)) + } + }) - metricQueryDuration = new prometheusClient.Summary({ - name: 'speckle_server_knex_query_duration', - help: 'Summary of the DB query durations in seconds' - }) + metricQueryDuration = new prometheusClient.Summary({ + name: 'speckle_server_knex_query_duration', + help: 'Summary of the DB query durations in seconds' + }) - metricQueryErrors = new prometheusClient.Counter({ - name: 'speckle_server_knex_query_errors', - help: 'Number of DB queries with errors' - }) + metricQueryErrors = new prometheusClient.Counter({ + name: 'speckle_server_knex_query_errors', + help: 'Number of DB queries with errors' + }) - knex.on('query', (data) => { - const queryId = data.__knexQueryUid + '' - queryStartTime[queryId] = Date.now() - }) + db.on('query', (data) => { + const queryId = data.__knexQueryUid + '' + queryStartTime[queryId] = Date.now() + }) - knex.on('query-response', (data, obj, builder) => { - const queryId = obj.__knexQueryUid + '' - const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 - delete queryStartTime[queryId] - if (!isNaN(durationSec)) metricQueryDuration.observe(durationSec) - }) + db.on('query-response', (data, obj, builder) => { + const queryId = obj.__knexQueryUid + '' + const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 + delete queryStartTime[queryId] + if (!isNaN(durationSec)) metricQueryDuration.observe(durationSec) + }) - knex.on('query-error', (err, querySpec) => { - const queryId = querySpec.__knexQueryUid + '' - const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 - delete queryStartTime[queryId] + db.on('query-error', (err, querySpec) => { + const queryId = querySpec.__knexQueryUid + '' + const durationSec = (Date.now() - queryStartTime[queryId]) / 1000 + delete queryStartTime[queryId] - if (!isNaN(durationSec)) metricQueryDuration.observe(durationSec) - metricQueryErrors.inc() - }) -} + if (!isNaN(durationSec)) metricQueryDuration.observe(durationSec) + metricQueryErrors.inc() + }) + } module.exports = { initPrometheusMetrics() { if (prometheusInitialized) return prometheusInitialized = true - initKnexPrometheusMetrics() + initDBPrometheusMetricsFactory({ db: knex })() // Define the HTTP server const server = http.createServer(async (req, res) => {