From a16ac938cc9fc5f0f96aef0bb1e934215ee0e1de Mon Sep 17 00:00:00 2001 From: "Jorge S. Mendes de Jesus" Date: Sun, 23 Feb 2020 08:34:35 +0100 Subject: [PATCH 1/5] where implementation on resulttype=hits --- pygeoapi/provider/postgresql.py | 59 +++++++++++++++++++------------ tests/test_postgresql_provider.py | 14 ++++++++ 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index bc3a53d..564255d 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -174,6 +174,35 @@ class PostgreSQLProvider(BaseProvider): self.fields = db.fields return self.fields + def __get_where_clauses(self, properties=[], bbox=[]): + """ + Generarates WHERE conditions to be implemented in query. + Private method mainly associated with query method + :param properties: list of tuples (name, value) + :param bbox: bounding box [minx,miny,maxx,maxy] + + :returns: psycopg2.sql.Composed or psycopg2.sql.SQL + """ + + where_conditions = [] + if properties: + property_clauses = [SQL('{} = {}').format( + Identifier(k), Literal(v)) for k, v in properties] + where_conditions += property_clauses + if bbox: + bbox_clause = SQL('{} && ST_MakeEnvelope({})').format( + Identifier(self.geom), SQL(', ').join( + [Literal(bbox_coord) for bbox_coord in bbox])) + where_conditions.append(bbox_clause) + + if where_conditions: + where_clause = SQL(' WHERE {}').format( + SQL(' AND ').join(where_conditions)) + else: + where_clause = SQL('') + + return where_clause + def query(self, startindex=0, limit=10, resulttype='results', bbox=[], datetime=None, properties=[], sortby=[]): """ @@ -198,8 +227,11 @@ class PostgreSQLProvider(BaseProvider): with DatabaseConnection(self.conn_dic, self.table, context="hits") as db: cursor = db.conn.cursor(cursor_factory=RealDictCursor) - sql_query = SQL("select count(*) as hits from {}").\ - format(Identifier(self.table)) + + where_clause = self.__get_where_clauses( + properties=properties, bbox=bbox) + sql_query = SQL("select count(*) as hits from {} {}").\ + format(Identifier(self.table), where_clause) try: cursor.execute(sql_query) except Exception as err: @@ -215,27 +247,10 @@ class PostgreSQLProvider(BaseProvider): with DatabaseConnection(self.conn_dic, self.table) as db: cursor = db.conn.cursor(cursor_factory=RealDictCursor) - where_conditions = [] - if properties: - property_clauses = \ - [SQL('{} = {}').format( - Identifier(k), Literal(v)) for k, v in properties] - where_conditions += property_clauses - if bbox: - bbox_clause = SQL('{} && ST_MakeEnvelope({})').format( - Identifier(self.geom), - SQL(', ').join( - [Literal(bbox_coord) for bbox_coord in bbox] - ) - ) - where_conditions.append(bbox_clause) - if where_conditions: - where_clause = SQL(' WHERE {}').format( - SQL(' AND ').join(where_conditions) - ) - else: - where_clause = SQL('') + where_clause = self.__get_where_clauses( + properties=properties, bbox=bbox) + sql_query = SQL("DECLARE \"geo_cursor\" CURSOR FOR \ SELECT {},ST_AsGeoJSON({}) FROM {}{}").\ format(db.columns, diff --git a/tests/test_postgresql_provider.py b/tests/test_postgresql_provider.py index f5097ca..94a9e05 100644 --- a/tests/test_postgresql_provider.py +++ b/tests/test_postgresql_provider.py @@ -88,6 +88,20 @@ def test_query_with_property_filter(config): assert (len(other_features) != 0) +def test_query_hits(config): + """Test query resulttype=hits with properties""" + psp = PostgreSQLProvider(config) + results = psp.query(resulttype="hits") + assert results["numberMatched"] == 14776 + + results = psp.query( + bbox=[29.3373, -3.4099, 29.3761, -3.3924], resulttype="hits") + assert results["numberMatched"] == 5 + + results = psp.query(properties=[("waterway", "stream")], resulttype="hits") + assert results["numberMatched"] == 13930 + + def test_query_bbox(config): """Test query with a specified bounding box""" psp = PostgreSQLProvider(config) From 982166728a78336192ed4b75ad4c0a3708b627a8 Mon Sep 17 00:00:00 2001 From: "Jorge S. Mendes de Jesus" Date: Mon, 24 Feb 2020 18:05:35 +0100 Subject: [PATCH 2/5] compatible with the commit 0ae2084 --- pygeoapi/provider/postgresql.py | 2 +- requirements-provider.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index 564255d..9a8b74d 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -252,7 +252,7 @@ class PostgreSQLProvider(BaseProvider): properties=properties, bbox=bbox) sql_query = SQL("DECLARE \"geo_cursor\" CURSOR FOR \ - SELECT {},ST_AsGeoJSON({}) FROM {}{}").\ + SELECT DISTINCT {},ST_AsGeoJSON({}) FROM {}{}").\ format(db.columns, Identifier(self.geom), Identifier(self.table), diff --git a/requirements-provider.txt b/requirements-provider.txt index 29c3121..dc964e0 100644 --- a/requirements-provider.txt +++ b/requirements-provider.txt @@ -1,4 +1,4 @@ elasticsearch==7.1.0 GDAL>=2.2,<3.0 -psycopg2==2.7.6 +psycopg2-binary==2.7.6 pymongo From eb2a92de0844ab90dbfcaab327ff171942779bb3 Mon Sep 17 00:00:00 2001 From: "Jorge S. Mendes de Jesus" Date: Mon, 24 Feb 2020 18:49:25 +0100 Subject: [PATCH 3/5] Revert "compatible with the commit 0ae2084" This reverts commit 982166728a78336192ed4b75ad4c0a3708b627a8. modified: pygeoapi/provider/postgresql.py modified: requirements-provider.txt --- pygeoapi/provider/postgresql.py | 2 +- requirements-provider.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index 9a8b74d..564255d 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -252,7 +252,7 @@ class PostgreSQLProvider(BaseProvider): properties=properties, bbox=bbox) sql_query = SQL("DECLARE \"geo_cursor\" CURSOR FOR \ - SELECT DISTINCT {},ST_AsGeoJSON({}) FROM {}{}").\ + SELECT {},ST_AsGeoJSON({}) FROM {}{}").\ format(db.columns, Identifier(self.geom), Identifier(self.table), diff --git a/requirements-provider.txt b/requirements-provider.txt index dc964e0..29c3121 100644 --- a/requirements-provider.txt +++ b/requirements-provider.txt @@ -1,4 +1,4 @@ elasticsearch==7.1.0 GDAL>=2.2,<3.0 -psycopg2-binary==2.7.6 +psycopg2==2.7.6 pymongo From 3caf4a92a4b888af0313d94dcef623a198bc1c1f Mon Sep 17 00:00:00 2001 From: "Jorge S. Mendes de Jesus" Date: Mon, 24 Feb 2020 19:00:36 +0100 Subject: [PATCH 4/5] distinct records on database --- pygeoapi/provider/postgresql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index 564255d..5937b7c 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -252,7 +252,7 @@ class PostgreSQLProvider(BaseProvider): properties=properties, bbox=bbox) sql_query = SQL("DECLARE \"geo_cursor\" CURSOR FOR \ - SELECT {},ST_AsGeoJSON({}) FROM {}{}").\ + SELECT DISTINCT {},ST_AsGeoJSON({}) FROM {}{}").\ format(db.columns, Identifier(self.geom), Identifier(self.table), @@ -337,8 +337,8 @@ class PostgreSQLProvider(BaseProvider): with DatabaseConnection(self.conn_dic, self.table) as db: cursor = db.conn.cursor(cursor_factory=RealDictCursor) - sql_query = SQL("select {},ST_AsGeoJSON({}) \ - from {} WHERE {}=%s").format(db.columns, + sql_query = SQL("SELECT {},ST_AsGeoJSON({}) \ + FROM {} WHERE {}=%s").format(db.columns, Identifier(self.geom), Identifier(self.table), Identifier(self.id_field)) From d007eb85cbc882e6d2bb8d79ccedaa1237c59464 Mon Sep 17 00:00:00 2001 From: "Jorge S. Mendes de Jesus" Date: Tue, 25 Feb 2020 08:56:06 +0100 Subject: [PATCH 5/5] Revert "distinct records on database" This reverts commit 3caf4a92a4b888af0313d94dcef623a198bc1c1f. modified: pygeoapi/provider/postgresql.py --- pygeoapi/provider/postgresql.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index 5937b7c..564255d 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -252,7 +252,7 @@ class PostgreSQLProvider(BaseProvider): properties=properties, bbox=bbox) sql_query = SQL("DECLARE \"geo_cursor\" CURSOR FOR \ - SELECT DISTINCT {},ST_AsGeoJSON({}) FROM {}{}").\ + SELECT {},ST_AsGeoJSON({}) FROM {}{}").\ format(db.columns, Identifier(self.geom), Identifier(self.table), @@ -337,8 +337,8 @@ class PostgreSQLProvider(BaseProvider): with DatabaseConnection(self.conn_dic, self.table) as db: cursor = db.conn.cursor(cursor_factory=RealDictCursor) - sql_query = SQL("SELECT {},ST_AsGeoJSON({}) \ - FROM {} WHERE {}=%s").format(db.columns, + sql_query = SQL("select {},ST_AsGeoJSON({}) \ + from {} WHERE {}=%s").format(db.columns, Identifier(self.geom), Identifier(self.table), Identifier(self.id_field))