gergo/pgTimeout (#805)

* feat(server): add server authz pipeline rework first sketch

* feat(server authz): add new server authz middleware poc implementation

* test(server authz): add unittests for the new server authz workflow

* feat(wip rework of fileuploads vs blob storage): add basim impl of separate blob storage service

* feat(fileimport service): refactored file import service to utilize the new asssetstorage service

* refactor(server errors): refactor server errors to use the shared module definitions

Now all the errors inherit from BaseError

* refactor(fileimport service): cleanup after refactor

* feat(frontend fileimports): use the new blob storage for downloading the original file

* refactor(server fileimports): clean up the remnants of S3 storage from file imports

* refactor(server authz): centralize generic authz pipeline configs

* refactor(server blob storage): refactor / rename everything to use the `blob-storage` name

* ci(circleci): add s3 objectstorage environment variables

* ci(circleci): fix missing env variables

* ci(circleci): add minio test container

* ci(circleci): fix minio app startup

* ci(circleci): enable circleci remote docker

* ci(circleci): fix minio startup

* ci(cirleci): detach and wait properly for minio to start

* ci(circleci): revert to additional minio img config, it only fails when the container is stopped ?!

* ci(circleci): disable file uploads

* fix(fileimports): update with blob storage refactor leftovers

* feat(server blob storage): add blob storage graphql api

* refactor(server errors): merge new errors to shared module

* fix(server comments rte): fix import for RTE error

* chore(fileimports): remove node-fetch from dependency

* chore(server): remove body parser dependency

* fix(server blob storage): fix gql api

* fix(frontend): fix fileupload item not loading the new upload status, cause of premature event fire

* feat(server blob storage): fix file size limit and allow for public streams

* Update packages/server/modules/blobstorage/graph/schemas/blobstorage.graphql

Co-authored-by: Kristaps Fabians Geikins <fabis94@live.com>

* chore(blobstorage): fix PR review issues

* fix(server): fix import bugs

* chore(docker): ignore python venv data for docker build context

* feat(knex): update knex configuration with min max connections and application_name

* feat(helm chart): configure postgres max connections for server in the helm chart

Co-authored-by: Kristaps Fabians Geikins <fabis94@live.com>

.dockerignore

@@ -12,6 +12,7 @@ ISSUE_TEMPLATE.md
 .mocharc.js
 readme.md
 **/Dockerfile
+**/.venv
 
 .pnp.*
 .yarn/*

packages/fileimport-service/knex.js

@@ -1,9 +1,15 @@
+/* eslint-disable camelcase */
 'use strict'
 
 module.exports = require('knex')({
   client: 'pg',
-  connection:
-    process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@localhost/speckle',
-  pool: { min: 1, max: 1 }
+  connection: {
+    application_name: 'speckle_fileimport_service',
+    connectionString:
+      process.env.PG_CONNECTION_STRING ||
+      'postgres://speckle:speckle@localhost/speckle',
+    query_timeout: 4.32e7
+  },
+  pool: { min: 0, max: 1 }
   // migrations are in managed in the server package
 })

packages/preview-service/knex.js

@@ -1,9 +1,15 @@
+/* eslint-disable camelcase */
 'use strict'
 
 module.exports = require('knex')({
   client: 'pg',
-  connection:
-    process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@localhost/speckle',
-  pool: { min: 1, max: 2 }
+  connection: {
+    application_name: 'speckle_preview_service',
+    connectionString:
+      process.env.PG_CONNECTION_STRING ||
+      'postgres://speckle:speckle@localhost/speckle',
+    query_timeout: 4.32e7
+  },
+  pool: { min: 0, max: 2 }
   // migrations are in managed in the server package
 })

packages/server/knexfile.js

@@ -1,3 +1,4 @@
+/* eslint-disable camelcase */
 /* istanbul ignore file */
 'use strict'
 
@@ -44,31 +45,47 @@ if (env.POSTGRES_USER && env.POSTGRES_PASSWORD) {
 // types.setTypeParser(TIMESTAMPTZ_OID, (val) => val)
 // types.setTypeParser(TIMESTAMP_OID, (val) => val)
 
+// Another NOTE:
+// this is why the new datetime columns are created like this
+// table.specificType('createdAt', 'TIMESTAMPTZ(3)').defaultTo(knex.fn.now())
+
+const postgresMaxConnections = env.POSTGRES_MAX_CONNECTIONS_SERVER || 4
+
+const commonConfig = {
+  client: 'pg',
+  migrations: {
+    directory: migrationDirs
+  },
+  pool: { min: 0, max: postgresMaxConnections }
+}
+
 /** @type {Object<string, import('knex').Knex.Config>} */
 const config = {
   test: {
-    client: 'pg',
-    connection: connectionUri || 'postgres://localhost/speckle2_test',
-    migrations: {
-      directory: migrationDirs
+    ...commonConfig,
+    connection: {
+      connectionString: connectionUri || 'postgres://localhost/speckle2_test',
+      application_name: 'speckle_server'
     }
   },
   development: {
-    client: 'pg',
-    connection: connectionUri || 'postgres://localhost/speckle2_dev',
-    migrations: {
-      directory: migrationDirs
-    },
-    pool: { min: 2, max: 4 }
+    ...commonConfig,
+    connection: {
+      connectionString: connectionUri || 'postgres://localhost/speckle2_dev',
+      application_name: 'speckle_server'
+    }
   },
   production: {
-    client: 'pg',
-    connection: connectionUri,
-    migrations: {
-      directory: migrationDirs
-    },
-    pool: { min: 2, max: 4 }
+    ...commonConfig,
+    connection: {
+      connectionString: connectionUri,
+      application_name: 'speckle_server',
+      // global timeout of 12 hours, that kills stuck connections
+      query_timeout: 4.32e7
+    }
   }
 }
 
+console.log(config)
+
 module.exports = config

packages/webhook-service/src/knex.js

@@ -1,9 +1,15 @@
+/* eslint-disable camelcase */
 'use strict'
 
 module.exports = require('knex')({
   client: 'pg',
-  connection:
-    process.env.PG_CONNECTION_STRING || 'postgres://speckle:speckle@localhost/speckle',
-  pool: { min: 1, max: 1 }
+  connection: {
+    application_name: 'speckle_webhook_service',
+    connectionString:
+      process.env.PG_CONNECTION_STRING ||
+      'postgres://speckle:speckle@localhost/speckle',
+    query_timeout: 4.32e7
+  },
+  pool: { min: 0, max: 1 }
   // migrations are in managed in the server package
 })

utils/helm/speckle-server/templates/deployment-backend.yml

@@ -107,6 +107,8 @@ spec:
               secretKeyRef:
                 name: {{ .Values.secretName }}
                 key: postgres_url
+          - name: POSTGRES_MAX_CONNECTIONS_SERVER
+            value: {{ .Values.db.maxConnectionsServer}}
 
           - name: PGSSLMODE
             value: "{{ .Values.db.PGSSLMODE }}"

utils/helm/speckle-server/values.yaml

@@ -8,6 +8,7 @@ docker_image_tag: v2.3.3
 db:
   # postgres_url: secret -> postgres_url
   useCertificate: false
+  maxConnectionsServer: 4
   certificate: '' # Multi-line string with the contents of `ca-certificate.crt`
   PGSSLMODE: require