diff --git a/docs/source/data-publishing/ogcapi-features.rst b/docs/source/data-publishing/ogcapi-features.rst index c9bb993..15ef485 100644 --- a/docs/source/data-publishing/ogcapi-features.rst +++ b/docs/source/data-publishing/ogcapi-features.rst @@ -344,6 +344,43 @@ Must have PostGIS installed. table: hotosm_bdi_waterways geom_field: foo_geom +A number of database connection options can be also configured in the provider in order to adjust properly the sqlalchemy engine client. +These are optional and if not specified, the default from the engine will be used. Please see also `SQLAlchemy docs `_. + +.. code-block:: yaml + + providers: + - type: feature + name: PostgreSQL + data: + host: 127.0.0.1 + port: 3010 # Default 5432 if not provided + dbname: test + user: postgres + password: postgres + search_path: [osm, public] + options: + # Maximum time to wait while connecting, in seconds. + connect_timeout: 10 + # Number of *milliseconds* that transmitted data may remain + # unacknowledged before a connection is forcibly closed. + tcp_user_timeout: 10000 + # Whether client-side TCP keepalives are used. 1 = use keepalives, + # 0 = don't use keepalives. + keepalives: 1 + # Number of seconds of inactivity after which TCP should send a + # keepalive message to the server. + keepalives_idle: 5 + # Number of TCP keepalives that can be lost before the client's + # connection to the server is considered dead. + keepalives_count: 5 + # Number of seconds after which a TCP keepalive message that is not + # acknowledged by the server should be retransmitted. + keepalives_interval: 1 + id_field: osm_id + table: hotosm_bdi_waterways + geom_field: foo_geom + The PostgreSQL provider is also able to connect to Cloud SQL databases. .. code-block:: yaml diff --git a/pygeoapi/config.py b/pygeoapi/config.py index 61e47a1..b5dae66 100644 --- a/pygeoapi/config.py +++ b/pygeoapi/config.py @@ -1,8 +1,10 @@ # ================================================================= # # Authors: Tom Kralidis +# Francesco Bartoli # # Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2023 Francesco Bartoli # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -57,6 +59,7 @@ def validate_config(instance_dict: dict) -> bool: :returns: `bool` of validation """ jsonschema_validate(json.loads(to_json(instance_dict)), load_schema()) + return True diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index 1524580..5b3ef28 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -103,7 +103,10 @@ class PostgreSQLProvider(BaseProvider): LOGGER.debug(f'Geometry field: {self.geom}') # Read table information from database - self._store_db_parameters(provider_def['data']) + options = None + if provider_def.get('options'): + options = provider_def['options'] + self._store_db_parameters(provider_def['data'], options) self._engine, self.table_model = self._get_engine_and_table_model() LOGGER.debug(f'DB connection: {repr(self._engine.url)}') self.fields = self.get_fields() @@ -267,13 +270,14 @@ class PostgreSQLProvider(BaseProvider): return feature - def _store_db_parameters(self, parameters): + def _store_db_parameters(self, parameters, options): self.db_user = parameters.get('user') self.db_host = parameters.get('host') self.db_port = parameters.get('port', 5432) self.db_name = parameters.get('dbname') self.db_search_path = parameters.get('search_path', ['public']) self._db_password = parameters.get('password') + self.db_options = options def _get_engine_and_table_model(self): """ @@ -296,10 +300,15 @@ class PostgreSQLProvider(BaseProvider): port=self.db_port, database=self.db_name ) + conn_args = { + 'client_encoding': 'utf8', + 'application_name': 'pygeoapi' + } + if self.db_options: + conn_args.update(self.db_options) engine = create_engine( conn_str, - connect_args={'client_encoding': 'utf8', - 'application_name': 'pygeoapi'}, + connect_args=conn_args, pool_pre_ping=True) _ENGINE_STORE[engine_store_key] = engine diff --git a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml index cbe664b..8326425 100644 --- a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml +++ b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml @@ -448,10 +448,13 @@ properties: options: type: object description: optional options key value pairs to pass to provider (i.e. GDAL creation) - patternProperties: - "^[a-z]{2}$": - allOf: - - type: string + oneOf: + - $ref: '#/definitions/provider/properties/PostgreSQL/properties/config/properties/options' + # - type: object + # patternProperties: + # "^[a-z]{2}$": + # allOf: + # - type: string properties: type: array description: only return the following properties, in order @@ -542,6 +545,26 @@ definitions: - type: array items: type: string + provider: + properties: + PostgreSQL: + properties: + config: + properties: + options: + properties: + connect_timeout: + type: integer + tcp_user_timeout: + type: integer + keepalives: + type: integer + keepalives_idle: + type: integer + keepalives_count: + type: integer + keepalives_interval: + type: integer required: - server - logging diff --git a/tests/pygeoapi-test-config-postgresql.yml b/tests/pygeoapi-test-config-postgresql.yml index 5ee6c9d..fafbbf4 100644 --- a/tests/pygeoapi-test-config-postgresql.yml +++ b/tests/pygeoapi-test-config-postgresql.yml @@ -1,8 +1,10 @@ # ================================================================= # # Authors: Tom Kralidis +# Francesco Bartoli # # Copyright (c) 2020 Tom Kralidis +# Copyright (c) 2023 Francesco Bartoli # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -132,6 +134,24 @@ resources: user: postgres password: postgres search_path: [osm, public] + options: + # Maximum time to wait while connecting, in seconds. + connect_timeout: 10 + # Number of *milliseconds* that transmitted data may remain + # unacknowledged before a connection is forcibly closed. + tcp_user_timeout: 10000 + # Whether client-side TCP keepalives are used. 1 = use keepalives, + # 0 = don't use keepalives. + keepalives: 1 + # Number of seconds of inactivity after which TCP should send a + # keepalive message to the server. + keepalives_idle: 5 + # Number of TCP keepalives that can be lost before the client's + # connection to the server is considered dead. + keepalives_count: 5 + # Number of seconds after which a TCP keepalive message that is not + # acknowledged by the server should be retransmitted. + keepalives_interval: 1 id_field: osm_id table: hotosm_bdi_waterways geom_field: foo_geom diff --git a/tests/test_postgresql_provider.py b/tests/test_postgresql_provider.py index 0bff90e..0501b65 100644 --- a/tests/test_postgresql_provider.py +++ b/tests/test_postgresql_provider.py @@ -76,6 +76,9 @@ def config(): 'password': PASSWORD, 'search_path': ['osm', 'public'] }, + 'options': { + 'connect_timeout': 10 + }, 'id_field': 'osm_id', 'table': 'hotosm_bdi_waterways', 'geom_field': 'foo_geom' @@ -90,6 +93,15 @@ def pg_api_(): return API(config) +def test_valid_connection_options(config): + if config.get('options'): + keys = list(config['options'].keys()) + for key in keys: + assert key in ['connect_timeout', 'tcp_user_timeout', 'keepalives', + 'keepalives_idle', 'keepalives_count', + 'keepalives_interval'] + + def test_query(config): """Testing query for a valid JSON object with geometry""" p = PostgreSQLProvider(config) @@ -254,24 +266,24 @@ def test_get_not_existing_item_raise_exception(config): @pytest.mark.parametrize('cql, expected_ids', [ - ("osm_id BETWEEN 80800000 AND 80900000", - [80827787, 80827793, 80835468, 80835470, 80835472, 80835474, - 80835475, 80835478, 80835483, 80835486]), - ("osm_id BETWEEN 80800000 AND 80900000 AND waterway = 'stream'", - [80835470]), - ("osm_id BETWEEN 80800000 AND 80900000 AND waterway ILIKE 'sTrEam'", - [80835470]), - ("osm_id BETWEEN 80800000 AND 80900000 AND waterway LIKE 's%'", - [80835470]), - ("osm_id BETWEEN 80800000 AND 80900000 AND name IN ('Muhira', 'Mpanda')", - [80835468, 80835472, 80835475, 80835478]), - ("osm_id BETWEEN 80800000 AND 80900000 AND name IS NULL", - [80835474, 80835483]), - ("osm_id BETWEEN 80800000 AND 80900000 AND BBOX(foo_geom, 29, -2.8, 29.2, -2.9)", # noqa - [80827793, 80835470, 80835472, 80835483, 80835489]), - ("osm_id BETWEEN 80800000 AND 80900000 AND " - "CROSSES(foo_geom, LINESTRING(29.091 -2.731, 29.253 -2.845))", - [80835470, 80835472, 80835489]) + ("osm_id BETWEEN 80800000 AND 80900000", + [80827787, 80827793, 80835468, 80835470, 80835472, 80835474, + 80835475, 80835478, 80835483, 80835486]), + ("osm_id BETWEEN 80800000 AND 80900000 AND waterway = 'stream'", + [80835470]), + ("osm_id BETWEEN 80800000 AND 80900000 AND waterway ILIKE 'sTrEam'", + [80835470]), + ("osm_id BETWEEN 80800000 AND 80900000 AND waterway LIKE 's%'", + [80835470]), + ("osm_id BETWEEN 80800000 AND 80900000 AND name IN ('Muhira', 'Mpanda')", + [80835468, 80835472, 80835475, 80835478]), + ("osm_id BETWEEN 80800000 AND 80900000 AND name IS NULL", + [80835474, 80835483]), + ("osm_id BETWEEN 80800000 AND 80900000 AND BBOX(foo_geom, 29, -2.8, 29.2, -2.9)", # noqa + [80827793, 80835470, 80835472, 80835483, 80835489]), + ("osm_id BETWEEN 80800000 AND 80900000 AND " + "CROSSES(foo_geom, LINESTRING(29.091 -2.731, 29.253 -2.845))", + [80835470, 80835472, 80835489]) ]) def test_query_cql(config, cql, expected_ids): """Test a variety of CQL queries"""