huanld/pygeoapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve postgres connection options (#1357)

Browse Source 
* Add database connection options

* Move under provider options

Add docs for connection options

Fix typo

Use already existing provider options from config json schema

Use already existing provider options from config json schema

* Validate configuration of provider options

* Replace pydantic validation with plain jsonschema
This commit is contained in:
Francesco Bartoli
2023-10-03 03:53:07 +02:00
committed by GitHub
parent 6eb272074b
commit 96cf72236a
6 changed files with 130 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/source/data-publishing/ogcapi-features.rst
+37
View File
@@ -344,6 +344,43 @@ Must have PostGIS installed.
table: hotosm_bdi_waterways
geom_field: foo_geom
A number of database connection options can be also configured in the provider in order to adjust properly the sqlalchemy engine client.
These are optional and if not specified, the default from the engine will be used. Please see also `SQLAlchemy docs <https://docs.sqlalchemy.org/en/14/core/engines.html#custom-dbapi-connect-arguments-on-connect-routines>`_.
.. code-block:: yaml
providers:
- type: feature
name: PostgreSQL
data:
host: 127.0.0.1
port: 3010 # Default 5432 if not provided
dbname: test
user: postgres
password: postgres
search_path: [osm, public]
options:
# Maximum time to wait while connecting, in seconds.
connect_timeout: 10
# Number of *milliseconds* that transmitted data may remain
# unacknowledged before a connection is forcibly closed.
tcp_user_timeout: 10000
# Whether client-side TCP keepalives are used. 1 = use keepalives,
# 0 = don't use keepalives.
keepalives: 1
# Number of seconds of inactivity after which TCP should send a
# keepalive message to the server.
keepalives_idle: 5
# Number of TCP keepalives that can be lost before the client's
# connection to the server is considered dead.
keepalives_count: 5
# Number of seconds after which a TCP keepalive message that is not
# acknowledged by the server should be retransmitted.
keepalives_interval: 1
id_field: osm_id
table: hotosm_bdi_waterways
geom_field: foo_geom
The PostgreSQL provider is also able to connect to Cloud SQL databases.
.. code-block:: yaml
pygeoapi/config.py
+3
View File
@@ -1,8 +1,10 @@
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
# Francesco Bartoli <xbartolone@gmail.com>
#
# Copyright (c) 2022 Tom Kralidis
# Copyright (c) 2023 Francesco Bartoli
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -57,6 +59,7 @@ def validate_config(instance_dict: dict) -> bool:
:returns: `bool` of validation
"""
jsonschema_validate(json.loads(to_json(instance_dict)), load_schema())
return True
pygeoapi/provider/postgresql.py
+13 -4
View File
@@ -103,7 +103,10 @@ class PostgreSQLProvider(BaseProvider):
LOGGER.debug(f'Geometry field: {self.geom}')
# Read table information from database
self._store_db_parameters(provider_def['data'])
options = None
if provider_def.get('options'):
options = provider_def['options']
self._store_db_parameters(provider_def['data'], options)
self._engine, self.table_model = self._get_engine_and_table_model()
LOGGER.debug(f'DB connection: {repr(self._engine.url)}')
self.fields = self.get_fields()
@@ -267,13 +270,14 @@ class PostgreSQLProvider(BaseProvider):
return feature
def _store_db_parameters(self, parameters):
def _store_db_parameters(self, parameters, options):
self.db_user = parameters.get('user')
self.db_host = parameters.get('host')
self.db_port = parameters.get('port', 5432)
self.db_name = parameters.get('dbname')
self.db_search_path = parameters.get('search_path', ['public'])
self._db_password = parameters.get('password')
self.db_options = options
def _get_engine_and_table_model(self):
"""
@@ -296,10 +300,15 @@ class PostgreSQLProvider(BaseProvider):
port=self.db_port,
database=self.db_name
)
conn_args = {
'client_encoding': 'utf8',
'application_name': 'pygeoapi'
}
if self.db_options:
conn_args.update(self.db_options)
engine = create_engine(
conn_str,
connect_args={'client_encoding': 'utf8',
'application_name': 'pygeoapi'},
connect_args=conn_args,
pool_pre_ping=True)
_ENGINE_STORE[engine_store_key] = engine
pygeoapi/schemas/config/pygeoapi-config-0.x.yml
+27 -4
View File
@@ -448,10 +448,13 @@ properties:
options:
type: object
description: optional options key value pairs to pass to provider (i.e. GDAL creation)
patternProperties:
"^[a-z]{2}$":
allOf:
- type: string
oneOf:
- $ref: '#/definitions/provider/properties/PostgreSQL/properties/config/properties/options'
# - type: object
# patternProperties:
# "^[a-z]{2}$":
# allOf:
# - type: string
properties:
type: array
description: only return the following properties, in order
@@ -542,6 +545,26 @@ definitions:
- type: array
items:
type: string
provider:
properties:
PostgreSQL:
properties:
config:
properties:
options:
properties:
connect_timeout:
type: integer
tcp_user_timeout:
type: integer
keepalives:
type: integer
keepalives_idle:
type: integer
keepalives_count:
type: integer
keepalives_interval:
type: integer
required:
- server
- logging
tests/pygeoapi-test-config-postgresql.yml
+20
View File
@@ -1,8 +1,10 @@
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
# Francesco Bartoli <xbartolone@gmail.com>
#
# Copyright (c) 2020 Tom Kralidis
# Copyright (c) 2023 Francesco Bartoli
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -132,6 +134,24 @@ resources:
user: postgres
password: postgres
search_path: [osm, public]
options:
# Maximum time to wait while connecting, in seconds.
connect_timeout: 10
# Number of *milliseconds* that transmitted data may remain
# unacknowledged before a connection is forcibly closed.
tcp_user_timeout: 10000
# Whether client-side TCP keepalives are used. 1 = use keepalives,
# 0 = don't use keepalives.
keepalives: 1
# Number of seconds of inactivity after which TCP should send a
# keepalive message to the server.
keepalives_idle: 5
# Number of TCP keepalives that can be lost before the client's
# connection to the server is considered dead.
keepalives_count: 5
# Number of seconds after which a TCP keepalive message that is not
# acknowledged by the server should be retransmitted.
keepalives_interval: 1
id_field: osm_id
table: hotosm_bdi_waterways
geom_field: foo_geom
tests/test_postgresql_provider.py
+30 -18
View File
@@ -76,6 +76,9 @@ def config():
'password': PASSWORD,
'search_path': ['osm', 'public']
},
'options': {
'connect_timeout': 10
},
'id_field': 'osm_id',
'table': 'hotosm_bdi_waterways',
'geom_field': 'foo_geom'
@@ -90,6 +93,15 @@ def pg_api_():
return API(config)
def test_valid_connection_options(config):
if config.get('options'):
keys = list(config['options'].keys())
for key in keys:
assert key in ['connect_timeout', 'tcp_user_timeout', 'keepalives',
'keepalives_idle', 'keepalives_count',
'keepalives_interval']
def test_query(config):
"""Testing query for a valid JSON object with geometry"""
p = PostgreSQLProvider(config)
@@ -254,24 +266,24 @@ def test_get_not_existing_item_raise_exception(config):
@pytest.mark.parametrize('cql, expected_ids', [
("osm_id BETWEEN 80800000 AND 80900000",
[80827787, 80827793, 80835468, 80835470, 80835472, 80835474,
80835475, 80835478, 80835483, 80835486]),
("osm_id BETWEEN 80800000 AND 80900000 AND waterway = 'stream'",
[80835470]),
("osm_id BETWEEN 80800000 AND 80900000 AND waterway ILIKE 'sTrEam'",
[80835470]),
("osm_id BETWEEN 80800000 AND 80900000 AND waterway LIKE 's%'",
[80835470]),
("osm_id BETWEEN 80800000 AND 80900000 AND name IN ('Muhira', 'Mpanda')",
[80835468, 80835472, 80835475, 80835478]),
("osm_id BETWEEN 80800000 AND 80900000 AND name IS NULL",
[80835474, 80835483]),
("osm_id BETWEEN 80800000 AND 80900000 AND BBOX(foo_geom, 29, -2.8, 29.2, -2.9)", # noqa
[80827793, 80835470, 80835472, 80835483, 80835489]),
("osm_id BETWEEN 80800000 AND 80900000 AND "
"CROSSES(foo_geom, LINESTRING(29.091 -2.731, 29.253 -2.845))",
[80835470, 80835472, 80835489])
("osm_id BETWEEN 80800000 AND 80900000",
[80827787, 80827793, 80835468, 80835470, 80835472, 80835474,
80835475, 80835478, 80835483, 80835486]),
("osm_id BETWEEN 80800000 AND 80900000 AND waterway = 'stream'",
[80835470]),
("osm_id BETWEEN 80800000 AND 80900000 AND waterway ILIKE 'sTrEam'",
[80835470]),
("osm_id BETWEEN 80800000 AND 80900000 AND waterway LIKE 's%'",
[80835470]),
("osm_id BETWEEN 80800000 AND 80900000 AND name IN ('Muhira', 'Mpanda')",
[80835468, 80835472, 80835475, 80835478]),
("osm_id BETWEEN 80800000 AND 80900000 AND name IS NULL",
[80835474, 80835483]),
("osm_id BETWEEN 80800000 AND 80900000 AND BBOX(foo_geom, 29, -2.8, 29.2, -2.9)", # noqa
[80827793, 80835470, 80835472, 80835483, 80835489]),
("osm_id BETWEEN 80800000 AND 80900000 AND "
"CROSSES(foo_geom, LINESTRING(29.091 -2.731, 29.253 -2.845))",
[80835470, 80835472, 80835489])
])
def test_query_cql(config, cql, expected_ids):
"""Test a variety of CQL queries"""