huanld/pygeoapi
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Issue 290 (#322)

Browse Source 
* property implementation on query method

* dataset with foo_geom as geom

* bbox search on sqlite3

* pytests for limit and property filter

* geopackage and sqlite3

* fix test on ogr_geopackage for new dataset version

* updated doc and removal of bbox cast

* err variable not used

* converstion of f string to .format()
This commit is contained in:
Jorge Samuel Mendes de Jesus
2020-01-02 17:22:27 +01:00
committed by Tom Kralidis
parent f0006c92f0
commit 0a46bf5054
12 changed files with 15410 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -111,3 +111,4 @@ openapi.yml
#Eclipse project
.project
docs/source/code.rst
+2 -11
View File
@@ -182,15 +182,6 @@ GeoJSON
:members:
:private-members:
GeoPackage
^^^^^^^^^^
.. automodule:: pygeoapi.provider.geopackage
:show-inheritance:
:members:
:private-members:
OGR
^^^
@@ -210,8 +201,8 @@ postgresql
:private-members:
sqlite
^^^^^^
sqlite/geopackage
^^^^^^^^^^^^^^^^^
.. automodule:: pygeoapi.provider.sqlite
:show-inheritance:
pygeoapi/plugin.py
+1 -1
View File
@@ -43,7 +43,7 @@ PLUGINS = {
'GeoPackage': 'pygeoapi.provider.geopackage.GeoPackageProvider',
'OGR': 'pygeoapi.provider.ogr.OGRProvider',
'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider',
'SQLite': 'pygeoapi.provider.sqlite.SQLiteProvider'
'SQLiteGPKG': 'pygeoapi.provider.sqlite.SQLiteGPKGProvider'
},
'formatter': {
'CSV': 'pygeoapi.formatter.csv_.CSVFormatter'
pygeoapi/provider/geopackage.py → pygeoapi/provider/__geopackage.py
View File
pygeoapi/provider/postgresql.py
+8 -3
View File
@@ -97,9 +97,9 @@ class DatabaseConnection(object):
try:
search_path = self.conn_dic.pop('search_path', ['public'])
if search_path != ['public']:
self.conn_dic["options"] = f'-c \
search_path={",".join(search_path)}'
LOGGER.debug(f'Using search path: {search_path} ')
self.conn_dic["options"] = '-c \
search_path={}'.format(",".join(search_path))
LOGGER.debug('Using search path: {} '.format(search_path))
self.conn = psycopg2.connect(**self.conn_dic)
except psycopg2.OperationalError:
@@ -164,6 +164,11 @@ class PostgreSQLProvider(BaseProvider):
self.get_fields()
def get_fields(self):
"""
Get fields from PostgreSQL table (columns are field)
:returns: dict of fields
"""
if not self.fields:
with DatabaseConnection(self.conn_dic, self.table) as db:
self.fields = db.fields
pygeoapi/provider/sqlite.py
+114 -44
View File
@@ -1,6 +1,6 @@
# =================================================================
#
# Authors: Jorge Samuel Mendes de Jesus <jorge.dejesus@geocat.net>
# Authors: Jorge Samuel Mendes de Jesus <jorge.dejesus@protonmail.net>
# Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Jorge Samuel Mendes de Jesus
@@ -39,15 +39,15 @@ from pygeoapi.provider.base import BaseProvider, ProviderConnectionError
LOGGER = logging.getLogger(__name__)
class SQLiteProvider(BaseProvider):
"""Generic provider for SQLITE using sqlite3 module.
class SQLiteGPKGProvider(BaseProvider):
"""Generic provider for SQLITE and GPKG using sqlite3 module.
This module requires install of libsqlite3-mod-spatialite
TODO: DELETE, UPDATE, CREATE
"""
def __init__(self, provider_def):
"""
SQLiteProvider Class constructor
SQLiteGPKGProvider Class constructor
:param provider_def: provider definitions from yml pygeoapi-config.
data,id_field, name set in parent class
@@ -57,6 +57,8 @@ class SQLiteProvider(BaseProvider):
BaseProvider.__init__(self, provider_def)
self.table = provider_def['table']
self.application_id = None
self.geom_col = None
LOGGER.debug('Setting SQLite properties:')
LOGGER.debug('Data source: {}'.format(self.data))
@@ -64,6 +66,29 @@ class SQLiteProvider(BaseProvider):
LOGGER.debug('ID_field: {}'.format(self.id_field))
LOGGER.debug('Table: {}'.format(self.table))
self.cursor = self.__load()
LOGGER.debug('Got cursor from DB')
LOGGER.debug('Get available fields/properties')
self.get_fields()
def get_fields(self):
"""
Get fields from sqlite table (columns are field)
:returns: dict of fields
"""
if not self.fields:
results = self.cursor.execute(
'PRAGMA table_info({})'.format(self.table)).fetchall()
[self.fields.update(
{item["name"]:item["type"].lower()}
) for item in results]
return self.fields
def __response_feature(self, row_data):
"""
Assembles GeoJSON output from DB query
@@ -78,7 +103,7 @@ class SQLiteProvider(BaseProvider):
'type': 'Feature'
}
feature["geometry"] = json.loads(
rd.pop('AsGeoJSON(geometry)')
rd.pop('AsGeoJSON({})'.format(self.geom_col))
)
feature['properties'] = rd
feature['id'] = feature['properties'].pop(self.id_field)
@@ -108,7 +133,8 @@ class SQLiteProvider(BaseProvider):
if (os.path.exists(self.data)):
conn = sqlite3.connect(self.data)
else:
raise InvalidPluginError
LOGGER.error('Path to sqlite does not exist')
raise InvalidPluginError()
try:
conn.enable_load_extension(True)
@@ -118,39 +144,74 @@ class SQLiteProvider(BaseProvider):
conn.row_factory = sqlite3.Row
conn.enable_load_extension(True)
# conn.set_trace_callback(LOGGER.debug)
cursor = conn.cursor()
try:
cursor.execute("SELECT load_extension('mod_spatialite.so')")
cursor.execute("PRAGMA table_info({})".format(self.table))
except sqlite3.OperationalError as err:
LOGGER.error('Extension loading error: {}'.format(err))
raise ProviderConnectionError()
result = cursor.fetchall()
# Checking for geopackage
cursor.execute("PRAGMA application_id")
result = cursor.fetchone()
self.application_id = result["application_id"]
if self.application_id == 1196444487:
LOGGER.info("Detected GPKG 1.2 and greater")
elif self.application_id == 1196437808:
LOGGER.info("Detected GPKG 1.0 or 1.1")
else:
LOGGER.info("No GPKG detected assuming spatial sqlite3")
self.application_id = 0
if self.application_id:
cursor.execute("SELECT AutoGPKGStart()")
result = cursor.fetchall()
if result[0][0] == 1:
LOGGER.info("Loaded Geopackage support")
else:
LOGGER.info("SELECT AutoGPKGStart() returned 0." +
"Detected GPKG but couldnt load support")
raise InvalidPluginError
if self.application_id:
self.geom_col = "geom"
else:
self.geom_col = "geometry"
try:
# TODO: Better exceptions declaring
# InvalidPluginError as Parent class
assert len(result), "Table not found"
assert len([item for item in result
if item['pk'] == 1]), "Primary key not found"
assert len([item for item in result
if self.id_field in item]), "id_field not present"
assert len([item for item in result
if 'GEOMETRY' in item]), "GEOMETRY column not found"
cursor.execute('PRAGMA table_info({})'.format(self.table))
result = cursor.fetchall()
except sqlite3.OperationalError:
LOGGER.error('Couldnt find table: {}'.format(self.table))
raise ProviderConnectionError()
except InvalidPluginError:
raise
try:
assert len(result), 'Table not found'
assert len([item for item in result
if self.id_field in item]), 'id_field not present'
self.columns = [item[1] for item in result if item[1] != 'GEOMETRY']
self.columns = ",".join(self.columns)+",AsGeoJSON(geometry)"
except AssertionError:
raise InvalidPluginError
self.columns = [item[1] for item in result if item[1] != self.geom_col]
self.columns = ','.join(self.columns)+',AsGeoJSON({})'.format(
self.geom_col)
if self.application_id:
self.table = "vgpkg_{}".format(self.table)
return cursor
def query(self, startindex=0, limit=10, resulttype='results',
bbox=[], datetime=None, properties=[], sortby=[]):
"""
Query SQLite for all the content.
Query SQLite/GPKG for all the content.
e,g: http://localhost:5000/collections/countries/items?
limit=1&resulttype=results
limit=5&startindex=2&resulttype=results&continent=Europe&admin=Albania&bbox=29.3373,-3.4099,29.3761,-3.3924
http://localhost:5000/collections/countries/items?continent=Africa&bbox=29.3373,-3.4099,29.3761,-3.3924
:param startindex: starting record to return (default 0)
:param limit: number of records to return (default 10)
@@ -162,30 +223,43 @@ class SQLiteProvider(BaseProvider):
:returns: GeoJSON FeaturesCollection
"""
LOGGER.debug('Querying SQLite')
cursor = self.__load()
LOGGER.debug('Got cursor from DB')
LOGGER.debug('Querying SQLite/GPKG')
if resulttype == 'hits':
res = cursor.execute("select count(*) as hits from {};".format(
self.table))
res = self.cursor.execute(
"select count(*) as hits from {};".format(self.table))
hits = res.fetchone()["hits"]
return self.__response_feature_hits(hits)
where_syntax = " where " if (properties or bbox) else ""
where_values = tuple()
if properties:
where_syntax += " and ".join(
["{}=?".format(k) for k, v in properties])
where_values += where_values + tuple((v for k, v in properties))
if bbox:
if properties:
where_syntax += " and "
# TODO: check name of geometry column
where_syntax += " Intersects({}, \
BuildMbr(?,?,?,?)) ".format(self.geom_col)
where_values += tuple(bbox)
sql_query = "select {} from \
{} {} limit ? offset ?".format(
self.columns, self.table, where_syntax)
end_index = startindex + limit
# Not working
# http://localhost:5000/collections/countries/items/?startindex=10
sql_query = "select {} from {} where rowid >= ? \
and rowid <= ?;".format(self.columns, self.table)
LOGGER.debug('SQL Query: {}'.format(sql_query))
LOGGER.debug('Start Index: {}'.format(startindex))
LOGGER.debug('End Index: {}'.format(end_index))
row_data = cursor.execute(sql_query, (startindex, end_index, ))
row_data = self.cursor.execute(
sql_query, where_values + (limit, startindex))
feature_collection = {
'type': 'FeatureCollection',
@@ -208,23 +282,19 @@ class SQLiteProvider(BaseProvider):
:returns: GeoJSON FeaturesCollection
"""
LOGGER.debug('Get item from SQLite')
LOGGER.debug('Get item from SQLite/GPKG')
cursor = self.__load()
LOGGER.debug('Got cursor from DB')
sql_query = "select {} from {} where {}==?;".format(self.columns,
self.table,
self.id_field)
sql_query = 'select {} from \
{} where {}==?;'.format(
self.columns, self.table, self.id_field)
LOGGER.debug('SQL Query: {}'.format(sql_query))
LOGGER.debug('Identifier: {}'.format(identifier))
row_data = cursor.execute(sql_query, (identifier, )).fetchone()
row_data = self.cursor.execute(sql_query, (identifier, )).fetchone()
feature = self.__response_feature(row_data)
return feature
def __repr__(self):
return '<SQLiteProvider> {}, {}'.format(self.data, self.table)
return '<SQLiteGPKGProvider> {}, {}'.format(self.data, self.table)
tests/data/hotosm_bdi_waterways.sql
+15106
View File
File diff suppressed because one or more lines are too long
tests/data/poi_portugal.gpkg
BIN
View File
Binary file not shown.
tests/test_geopackage_provider.py
-73
View File
@@ -1,73 +0,0 @@
# =================================================================
#
# Authors: Just van den Broecke <justb4@gmail.com>
# Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2019 Just van den Broecke
# Copyright (c) 2019 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
# Needs to be run like: pytest -s test_geopackage_provider.py
# In eclipse we need to set PYGEOAPI_CONFIG, Run>Debug Configurations>
# (Arguments as py.test and set external variables to the correct config path)
import pytest
from pygeoapi.provider.geopackage import GeoPackageProvider
@pytest.fixture()
def config():
return {
'name': 'GeoPackage',
'data': './tests/data/poi_portugal.gpkg',
'id_field': 'osm_id',
'table': 'poi_portugal'
}
def test_query(config):
"""Testing query for a valid JSON object with geometry"""
p = GeoPackageProvider(config)
feature_collection = p.query()
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert features is not None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
def test_get(config):
p = GeoPackageProvider(config)
result = p.get(5156778016)
assert isinstance(result, dict)
assert 'geometry' in result
assert 'properties' in result
assert 'id' in result
assert 'tourist_info' in result['properties']['fclass']
tests/test_ogr_gpkg_provider.py
+3 -3
View File
@@ -73,9 +73,9 @@ def test_query(config_poi_portugal):
def test_get(config_poi_portugal):
p = OGRProvider(config_poi_portugal)
result = p.get(5156778016)
assert result['id'] == 5156778016
assert 'tourist_info' in result['properties']['fclass']
result = p.get(536678593)
assert result['id'] == 536678593
assert 'cafe' in result['properties']['fclass']
# Testing with GeoPackage files with identical features
tests/test_sqlite_geopackage_provider.py
+175
View File
@@ -0,0 +1,175 @@
# =================================================================
#
# Authors: Just van den Broecke <justb4@gmail.com>
# Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2019 Just van den Broecke
# Copyright (c) 2019 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
# Needs to be run like: pytest -s test_sqlite_provider.py
# In eclipse we need to set PYGEOAPI_CONFIG, Run>Debug Configurations>
# (Arguments as py.test and set external variables to the correct config path)
import pytest
from pygeoapi.provider.sqlite import SQLiteGPKGProvider
@pytest.fixture()
def config_sqlite():
return {
'name': 'Sqlite',
'data': './tests/data/ne_110m_admin_0_countries.sqlite',
'id_field': 'ogc_fid',
'table': 'ne_110m_admin_0_countries'
}
@pytest.fixture()
def config_geopackage():
return {
'name': 'GeoPackage',
'data': './tests/data/poi_portugal.gpkg',
'id_field': 'osm_id',
'table': 'poi_portugal'
}
def test_query_sqlite(config_sqlite):
"""Testing query for a valid JSON object with geometry for sqlite3"""
p = SQLiteGPKGProvider(config_sqlite)
feature_collection = p.query()
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert features is not None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
def test_query_geopackage(config_geopackage):
"""Testing query for a valid JSON object with geometry for geopackage"""
p = SQLiteGPKGProvider(config_geopackage)
feature_collection = p.query()
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert features is not None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
def test_query_with_property_filter_sqlite(config_sqlite):
"""Test query valid features when filtering by property"""
p = SQLiteGPKGProvider(config_sqlite)
feature_collection = p.query(properties=[
("continent", "Europe")], limit=100)
features = feature_collection.get('features', None)
assert len(features) == 39
def test_query_with_property_filter_geopackage(config_geopackage):
"""Test query valid features when filtering by property"""
p = SQLiteGPKGProvider(config_geopackage)
feature_collection = p.query(properties=[
("fclass", "cafe")], limit=10000)
features = feature_collection.get('features', None)
assert len(features) == 823
def test_query_with_property_filter_bbox_sqlite(config_sqlite):
"""Test query valid features when filtering by property"""
p = SQLiteGPKGProvider(config_sqlite)
feature_collection = p.query(properties=[("continent", "Europe")],
bbox=[29.3373, -3.4099, 29.3761, -3.3924])
features = feature_collection.get('features', None)
assert len(features) == 0
def test_query_with_property_filter_bbox_geopackage(config_geopackage):
"""Test query valid features when filtering by property"""
p = SQLiteGPKGProvider(config_geopackage)
feature_collection = p.query(properties=[("fclass", "cafe")],
bbox=[
-16.3991310876,
33.0063015781,
-16.3366454278,
33.0560854323
])
features = feature_collection.get('features', None)
assert len(features) == 0
def test_query_bbox_sqlite(config_sqlite):
"""Test query with a specified bounding box"""
psp = SQLiteGPKGProvider(config_sqlite)
boxed_feature_collection = psp.query(
bbox=[29.3373, -3.4099, 29.3761, -3.3924]
)
assert len(boxed_feature_collection['features']) == 1
assert 'Burundi' in \
boxed_feature_collection['features'][0]['properties']['name']
def test_query_bbox_geopackage(config_geopackage):
"""Test query with a specified bounding box"""
psp = SQLiteGPKGProvider(config_geopackage)
boxed_feature_collection = psp.query(
bbox=[-16.3991310876, 33.0063015781, -16.3366454278, 33.0560854323]
)
assert len(boxed_feature_collection['features']) == 5
def test_get_sqlite(config_sqlite):
p = SQLiteGPKGProvider(config_sqlite)
result = p.get(118)
assert isinstance(result, dict)
assert 'geometry' in result
assert 'properties' in result
assert 'id' in result
assert 'Netherlands' in result['properties']['admin']
def test_get_geopackage(config_geopackage):
p = SQLiteGPKGProvider(config_geopackage)
result = p.get(536678593)
assert isinstance(result, dict)
assert 'geometry' in result
assert 'properties' in result
assert 'id' in result
assert 'Académico' in result['properties']['name']
tests/test_sqlite_provider.py
-72
View File
@@ -1,72 +0,0 @@
# =================================================================
#
# Authors: Just van den Broecke <justb4@gmail.com>
# Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2019 Just van den Broecke
# Copyright (c) 2019 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
# Needs to be run like: pytest -s test_sqlite_provider.py
# In eclipse we need to set PYGEOAPI_CONFIG, Run>Debug Configurations>
# (Arguments as py.test and set external variables to the correct config path)
import pytest
from pygeoapi.provider.sqlite import SQLiteProvider
@pytest.fixture()
def config():
return {
'name': 'Sqlite',
'data': './tests/data/ne_110m_admin_0_countries.sqlite',
'id_field': 'ogc_fid',
'table': 'ne_110m_admin_0_countries'
}
def test_query(config):
"""Testing query for a valid JSON object with geometry"""
p = SQLiteProvider(config)
feature_collection = p.query()
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert features is not None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
def test_get(config):
p = SQLiteProvider(config)
result = p.get(118)
assert isinstance(result, dict)
assert 'geometry' in result
assert 'properties' in result
assert 'id' in result
assert 'Netherlands' in result['properties']['admin']