feat(db monitor): adds a new metric speckle_db_tablesize with db table sizes (#2474)

* feat(db queries): adds db query scripts for determining db sizes

* Adds a new metric `speckle_db_tablesize` with db table sizes

* lower case all the file types to provide a combined metric

* group file status count by lower cased file type

* reinstate accidentally deleted metric

* Fix developer script

.gitignore

@@ -15,7 +15,6 @@ packages/*/dist-cjs
 .nyc_output
 coverage/
 .idea
-test-queries
 
 **/.DS_Store
 .nvmrc

test-queries/all-tables-sizes.sql

@@ -0,0 +1,18 @@
+SELECT
+  schema_name,
+  relname,
+  pg_size_pretty(table_size) AS size,
+  table_size
+
+FROM (
+       SELECT
+         pg_catalog.pg_namespace.nspname           AS schema_name,
+         relname,
+         pg_relation_size(pg_catalog.pg_class.oid) AS table_size
+
+       FROM pg_catalog.pg_class
+         JOIN pg_catalog.pg_namespace ON relnamespace = pg_catalog.pg_namespace.oid
+     ) t
+WHERE schema_name NOT LIKE 'pg_%'
+AND schema_name NOT LIKE 'information_schema'
+ORDER BY table_size DESC;

test-queries/objects-sizes.sql

@@ -0,0 +1,3 @@
+SELECT id, "streamId", "speckleType", "createdAt"::date, pg_column_size(data) / 1024 /1024 || 'MB' AS size, data
+	FROM objects
+	ORDER BY size DESC LIMIT 20;

utils/monitor-deployment/dev_run.sh

@@ -1,4 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
+set -eo pipefail
+
+GIT_ROOT="$(git rev-parse --show-toplevel)"
 
 export PG_CONNECTION_STRING=postgres://speckle:speckle@localhost/speckle
-cd src && python3 -u run.py
+pushd "${GIT_ROOT}/utils/monitor-deployment"
+trap popd EXIT
+pip install --disable-pip-version-check --requirement ./requirements.txt
+python3 -u src/run.py

utils/monitor-deployment/src/run.py

@@ -61,6 +61,11 @@ PROM = {
         "Size of imported files, by type (in bytes)",
         labelnames=("filetype",),
     ),
+    "tablesize": Gauge(
+        "speckle_db_tablesize",
+        "Size of tables in the database, by table (in bytes)",
+        labelnames=("table",),
+    ),
 }
 
 
@@ -82,9 +87,9 @@ def tick(cur):
     # File Imports
     cur.execute(
         """
-        SELECT "fileType", "convertedStatus", count(*)
+        SELECT LOWER("fileType"), "convertedStatus", count(*)
-        FROM file_uploads
+          FROM file_uploads
-        GROUP BY ("fileType", "convertedStatus")
+          GROUP BY (LOWER("fileType"), "convertedStatus");
         """
     )
     # put in a dictionary so we fill non-existing statuses with zeroes
@@ -105,9 +110,9 @@ def tick(cur):
 
     cur.execute(
         """
-        SELECT "fileType", SUM("fileSize")
+        SELECT LOWER("fileType") AS fileType, SUM("fileSize") AS fileSize
-        FROM file_uploads
+            FROM file_uploads
-        GROUP BY "fileType"
+            GROUP BY LOWER("fileType");
         """
     )
     for row in cur:
@@ -147,6 +152,30 @@ def tick(cur):
         else:
             PROM["previews"].labels(str(status)).set(0)
 
+    # Table sizes
+    cur.execute(
+        """
+      SELECT
+        relname,
+        table_size
+
+      FROM (
+            SELECT
+              pg_catalog.pg_namespace.nspname           AS schema_name,
+              relname,
+              pg_relation_size(pg_catalog.pg_class.oid) AS table_size
+
+            FROM pg_catalog.pg_class
+              JOIN pg_catalog.pg_namespace ON relnamespace = pg_catalog.pg_namespace.oid
+          ) t
+      WHERE schema_name = 'public'
+      ORDER BY table_size DESC;
+      """
+    )
+    values = {}
+    for row in cur:
+        PROM["tablesize"].labels(row[0]).set(row[1])
+
 
 def main():
     start_http_server(9092)