Issue #127 ogrpaging for all OGR source types (#128)

* #127 Paging for ALL OGR Drivers, tested: GPKG, Shapefile, SQLite, GeoJSON

* #127 Paging for ALL OGR Drivers, tested: GPKG, Shapefile, SQLite, GeoJSON fix SQLite Test

* #127 Paging for ALL OGR Drivers require GDAL greater or eq 2.2

* geopython/pygeoapi#112 geopython/pygeoapi#127 close OGR  ResultSet when layer from ExecuteSQL created

* geopython/pygeoapi#112 geopython/pygeoapi#127 grr flake8 fix
This commit is contained in:
Just van den Broecke
2019-05-21 17:34:54 +02:00
committed by Tom Kralidis
parent d2abc4873d
commit e9fc22e72c
7 changed files with 512 additions and 45 deletions
+41
View File
@@ -273,6 +273,47 @@ datasets:
id_field: gml_id
layer: rdinfo:stations
ogr_addresses_sqlite:
# Gotcha: don't use the name 'ogc_fid' as id-field in your SQLite tables:
# OGR will not show this field within GeoJSON properties dict.
title: Dutch addresses (subset Otterlo). OGR SQLite Driver
description: Dutch addresses subset.
keywords:
- Netherlands
- addresses
- INSPIRE
crs:
- CRS84
links:
- type: text/html
rel: canonical
title: information
href: http://www.nationaalgeoregister.nl/geonetwork/srv/dut/catalog.search#/metadata/4074b3c3-ca85-45ad-bc0d-b5fca8540z0b
hreflang: nl-NL
extents:
spatial:
bbox: [50.7539, 7.21097, 53.4658, 3.37087]
temporal:
begin: None
end: now # or empty
provider:
name: OGR
data:
source_type: SQLite
source: tests/data/dutch_addresses_4326.sqlite
source_srs: EPSG:4326
target_srs: EPSG:4326
source_capabilities:
paging: True
gdal_ogr_options:
EMPTY_AS_NULL: NO
GDAL_CACHEMAX: 64
CPL_DEBUG: NO
id_field: id
layer: ogrgeojson
processes:
hello-world:
processor:
+142 -40
View File
@@ -40,14 +40,25 @@ LOGGER = logging.getLogger(__name__)
class OGRProvider(BaseProvider):
"""OGR Provider"""
"""
OGR Provider. Uses GDAL/OGR Python-bindings to access OGR
Vector sources. References:
https://pcjericks.github.io/py-gdalogr-cookbook/
https://www.gdal.org/ogr_formats.html (per-driver specifics).
In theory any OGR source type (Driver) could be used, although
some Source Types are Driver-specific handling. This is handled
in Source Helper classes, instantiated per Source-Type.
The following Source Types have been tested to work:
GeoPackage (GPKG), SQLite, GeoJSON, ESRI Shapefile, WFS v2.
"""
# To deal with some OGR Source-Driver specifics.
SOURCE_HELPERS = {
'WFS': 'pygeoapi.provider.ogr.WFSHelper',
'ESRI Shapefile': 'pygeoapi.provider.ogr.ShapefileHelper',
'ESRIJSON': 'pygeoapi.provider.ogr.ESRIJSONHelper',
'GPKG': 'pygeoapi.provider.ogr.GPKGHelper'
'WFS': 'pygeoapi.provider.ogr.WFSHelper',
'*': 'pygeoapi.provider.ogr.CommonSourceHelper'
}
def __init__(self, provider_def):
@@ -152,10 +163,16 @@ class OGRProvider(BaseProvider):
self._load_source_helper(self.data_def['source_type'])
# Init
# Layer name is required
self.layer_name = provider_def.get('layer', None)
if not self.layer_name:
msg = 'Need explicit \'layer\' attr in provider config'
LOGGER.error(msg)
raise Exception(msg)
# Init driver and Source connection
self.driver = None
self.conn = None
self.layer_name = provider_def.get('layer', None)
def _open(self):
source_type = self.data_def['source_type']
@@ -176,6 +193,7 @@ class OGRProvider(BaseProvider):
self.source_helper.disable_paging()
def _close(self):
self.source_helper.close()
self.conn = None
LOGGER.debug('closed self.conn')
@@ -185,18 +203,8 @@ class OGRProvider(BaseProvider):
if not self.conn:
self._open()
if not self.layer_name:
# E.g. Shapefiles may not have explicitly named Layers
layer = self.conn.GetLayer(0)
else:
layer = self.conn.GetLayerByName(self.layer_name)
if not layer:
msg = 'Cannot get Layer {} from OGR Source'.format(self.layer_name)
LOGGER.error(msg)
raise Exception(msg)
return layer
# Delegate getting Layer to SourceHelper
return self.source_helper.get_layer()
def get_fields(self):
"""
@@ -265,6 +273,8 @@ class OGRProvider(BaseProvider):
# layer.SetSpatialFilterRect(
# float(minx), float(miny), float(maxx), float(maxy))
# Make response based on resulttype specified
if resulttype == 'hits':
LOGGER.debug('hits only specified')
result = self._response_feature_hits(layer)
@@ -319,14 +329,12 @@ class OGRProvider(BaseProvider):
:returns: Source Helper object
"""
helper_type = source_type
if source_type not in OGRProvider.SOURCE_HELPERS.keys():
msg = 'No Helper found for OGR Source type: {}'.format(source_type)
LOGGER.exception(msg)
raise InvalidHelperError(msg)
helper_type = '*'
# Create object from full package.class name string.
source_helper_class = OGRProvider.SOURCE_HELPERS[source_type]
source_helper_class = OGRProvider.SOURCE_HELPERS[helper_type]
packagename, classname = source_helper_class.rsplit('.', 1)
module = importlib.import_module(packagename)
@@ -340,7 +348,11 @@ class OGRProvider(BaseProvider):
geom.Transform(self.transform_out)
json_feature = ogr_feature.ExportToJson(as_object=True)
json_feature['id'] = json_feature['properties'].pop(self.id_field)
try:
json_feature['id'] = json_feature['properties'].pop(self.id_field)
except Exception:
json_feature['id'] = ogr_feature.GetFID()
return json_feature
def _response_feature_collection(self, layer, limit):
@@ -397,9 +409,15 @@ class InvalidHelperError(Exception):
class SourceHelper:
"""
Helper classes for OGR-specific Source Types (Drivers).
For some actions Driver-specific settings or processing is
required. This is delegated to the OGR SourceHelper classes.
"""
def __init__(self, provider):
"""
Initialize object
Initialize object with related OGRProvider object.
:param provider: provider instance
@@ -407,6 +425,31 @@ class SourceHelper:
"""
self.provider = provider
def close(self):
"""
OGR Driver-specific handling of closing dataset.
Default is no specific handling.
"""
pass
def get_layer(self):
"""
Default action to get a Layer object from opened OGR Driver.
:return:
"""
layer = self.provider.conn.GetLayerByName(self.provider.layer_name)
if not layer:
msg = 'Cannot get Layer {} from OGR Source'.\
format(self.provider.layer_name)
LOGGER.error(msg)
raise Exception(msg)
return layer
def enable_paging(self, startindex=-1, limit=-1):
"""
Enable paged access to dataset (OGR Driver-specific)
@@ -423,8 +466,11 @@ class SourceHelper:
pass
class GPKGHelper(SourceHelper):
class CommonSourceHelper(SourceHelper):
"""
SourceHelper for most common OGR Source types:
Shapefile, GeoPackage, SQLite, GeoJSON etc.
"""
def __init__(self, provider):
"""
Initialize object
@@ -433,22 +479,80 @@ class GPKGHelper(SourceHelper):
:returns: pygeoapi.providers.ogr.SourceHelper
"""
self.provider = provider
SourceHelper.__init__(self, provider)
self.startindex = -1
self.limit = -1
self.result_set = None
class ShapefileHelper(SourceHelper):
def __init__(self, provider):
def close(self):
"""
Initialize object
OGR Driver-specific handling of closing dataset.
If ExecuteSQL has been (successfully) called
must close ResultSet explicitly.
https://gis.stackexchange.com/questions/114112/
explicitly-close-a-ogr-result-object-from-a-call-to-executesql
:param provider: provider instance
:returns: pygeoapi.providers.ogr.SourceHelper
"""
self.provider = provider
SourceHelper.__init__(self, provider)
if not self.result_set:
return
try:
self.provider.conn.ReleaseResultSet(self.result_set)
except Exception as err:
msg = 'ReleaseResultSet exception for Layer {}'.format(
self.provider.layer_name)
LOGGER.error(msg, err)
finally:
self.result_set = None
def enable_paging(self, startindex=-1, limit=-1):
"""
Enable paged access to dataset (OGR Driver-specific)
using OGR SQL https://www.gdal.org/ogr_sql.html
e.g. SELECT * FROM poly LIMIT 10 OFFSET 30
"""
self.startindex = startindex
self.limit = limit
def disable_paging(self):
"""
Disable paged access to dataset (OGR Driver-specific)
"""
pass
def get_layer(self):
"""
Gets OGR Layer from opened OGR dataset.
When startindex defined 1 or greater will invoke
OGR SQL SELECT with LIMIT and OFFSET and return
as Layer as ResultSet from ExecuteSQL on dataset.
:return: OGR layer object
"""
if self.startindex <= 0:
return SourceHelper.get_layer(self)
self.close()
sql = "SELECT * FROM {ds_name} LIMIT {limit} OFFSET {offset}".format(
ds_name=self.provider.layer_name,
limit=self.limit,
offset=self.startindex)
self.result_set = self.provider.conn.ExecuteSQL(sql)
# Reset since needs to be set each time explicitly
self.startindex = -1
self.limit = -1
if not self.result_set:
msg = 'Cannot get Layer {} via ExecuteSQL'.format(
self.provider.layer_name)
LOGGER.error(msg)
raise Exception(msg)
return self.result_set
class ESRIJSONHelper(SourceHelper):
@@ -461,7 +565,6 @@ class ESRIJSONHelper(SourceHelper):
:returns: pygeoapi.providers.ogr.SourceHelper
"""
self.provider = provider
SourceHelper.__init__(self, provider)
def enable_paging(self, startindex=-1, limit=-1):
@@ -501,7 +604,6 @@ class WFSHelper(SourceHelper):
:returns: pygeoapi.providers.ogr.SourceHelper
"""
self.provider = provider
SourceHelper.__init__(self, provider)
def enable_paging(self, startindex=-1, limit=-1):
+1 -1
View File
@@ -15,4 +15,4 @@ pytest-env
pyOpenSSL==17.5.0
ndg-httpsclient==0.4.4
pyasn1==0.4.2
GDAL>=2.1
GDAL>=2.2
Binary file not shown.
+86 -2
View File
@@ -67,7 +67,8 @@ def config_gpkg_4326():
'paging': True
},
},
'id_field': 'id'
'id_field': 'id',
'layer': 'OGRGeoJSON'
}
@@ -86,7 +87,8 @@ def config_gpkg_28992():
'paging': True
},
},
'id_field': 'id'
'id_field': 'id',
'layer': 'OGRGeoJSON'
}
@@ -249,3 +251,85 @@ def test_query_with_limit_4326(config_gpkg_4326):
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
def test_query_with_startindex_28992(config_gpkg_28992):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_gpkg_28992)
feature_collection = p.query(startindex=20, limit=5, resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 5
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
assert feature['id'] == 'inspireadressen.1744969'
assert 'Egypte' in properties['straatnaam']
geometry = feature.get('geometry', None)
assert geometry is not None
def test_query_with_startindex_4326(config_gpkg_4326):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_gpkg_4326)
feature_collection = p.query(startindex=20, limit=5, resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 5
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
assert feature['id'] == 'inspireadressen.1744969'
assert 'Egypte' in properties['straatnaam']
geometry = feature.get('geometry', None)
assert geometry is not None
def test_query_bbox_with_startindex_28992(config_gpkg_28992):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_gpkg_28992)
feature_collection = p.query(
startindex=10, limit=5,
bbox=(5.742, 52.053, 5.773, 52.098),
resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 5
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
assert properties['straatnaam'] == 'Buurtweg'
assert properties['huisnummer'] == '4'
def test_query_bbox_with_startindex_4326(config_gpkg_4326):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_gpkg_4326)
feature_collection = p.query(
startindex=1, limit=5,
bbox=(5.742, 52.053, 5.773, 52.098),
resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 5
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
assert properties['straatnaam'] == 'Egypte'
assert properties['huisnummer'] == '6'
+86 -2
View File
@@ -25,7 +25,8 @@ def config_shapefile_4326():
'paging': True
},
},
'id_field': 'id'
'id_field': 'id',
'layer': 'inspireadressen'
}
@@ -44,7 +45,8 @@ def config_shapefile_28992():
'paging': True
},
},
'id_field': 'id'
'id_field': 'id',
'layer': 'inspireadressen'
}
@@ -207,3 +209,85 @@ def test_query_with_limit_4326(config_shapefile_4326):
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
def test_query_with_startindex_28992(config_shapefile_28992):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_shapefile_28992)
feature_collection = p.query(startindex=20, limit=5, resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 5
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
assert feature['id'] == 'inspireadressen.1744969'
assert 'Egypte' in properties['straatnaam']
geometry = feature.get('geometry', None)
assert geometry is not None
def test_query_with_startindex_4326(config_shapefile_4326):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_shapefile_4326)
feature_collection = p.query(startindex=20, limit=5, resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 5
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
assert feature['id'] == 'inspireadressen.1744969'
assert 'Egypte' in properties['straatnaam']
geometry = feature.get('geometry', None)
assert geometry is not None
def test_query_bbox_with_startindex_28992(config_shapefile_28992):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_shapefile_28992)
feature_collection = p.query(
startindex=10, limit=5,
bbox=(5.742, 52.053, 5.773, 52.098),
resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 5
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
assert properties['straatnaam'] == 'Buurtweg'
assert properties['huisnummer'] == '4'
def test_query_bbox_with_startindex_4326(config_shapefile_4326):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_shapefile_4326)
feature_collection = p.query(
startindex=1, limit=5,
bbox=(5.742, 52.053, 5.773, 52.098),
resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 5
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
assert properties['straatnaam'] == 'Egypte'
assert properties['huisnummer'] == '6'
+156
View File
@@ -0,0 +1,156 @@
# Needs to be run like: python3 -m pytest
import logging
import pytest
from pygeoapi.provider.ogr import OGRProvider
LOGGER = logging.getLogger(__name__)
# Testing with SQLite files with identical features
# (all 2481 addresses in Otterlo Netherlands).
@pytest.fixture()
def config_sqlite_4326():
return {
'name': 'OGR',
'data': {
'source_type': 'SQLite',
'source':
'./tests/data/dutch_addresses_4326.sqlite',
'source_srs': 'EPSG:4326',
'target_srs': 'EPSG:4326',
'source_capabilities': {
'paging': True
},
},
'id_field': 'id',
'layer': 'ogrgeojson'
}
def test_get_fields_4326(config_sqlite_4326):
"""Testing field types"""
p = OGRProvider(config_sqlite_4326)
results = p.get_fields()
assert results['straatnaam'] == 'string'
assert results['huisnummer'] == 'string'
def test_get_4326(config_sqlite_4326):
"""Testing query for a specific object"""
p = OGRProvider(config_sqlite_4326)
result = p.get('inspireadressen.1747652')
assert result['id'] == 'inspireadressen.1747652'
assert 'Mosselsepad' in result['properties']['straatnaam']
def test_query_hits_4326(config_sqlite_4326):
"""Testing query on entire collection for hits"""
p = OGRProvider(config_sqlite_4326)
feature_collection = p.query(resulttype='hits')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) is 0
hits = feature_collection.get('numberMatched', None)
assert hits is not None
assert hits == 2481
def test_query_bbox_hits_4326(config_sqlite_4326):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_sqlite_4326)
# feature_collection = p.query(
# bbox=[120000, 480000, 124000, 487000], resulttype='hits')
feature_collection = p.query(
bbox=[5.763409, 52.060197, 5.769256, 52.061976], resulttype='hits')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) is 0
hits = feature_collection.get('numberMatched', None)
assert hits is not None
print('hits={}'.format(hits))
assert hits is 1
def test_query_bbox_4326(config_sqlite_4326):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_sqlite_4326)
# feature_collection = p.query(
# bbox=[180800, 452500, 181200, 452700], resulttype='results')
feature_collection = p.query(
bbox=(5.763409, 52.060197, 5.769256, 52.061976), resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 1
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
assert properties['straatnaam'] == 'Planken Wambuisweg'
def test_query_with_limit_4326(config_sqlite_4326):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_sqlite_4326)
feature_collection = p.query(limit=5, resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 5
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
def test_query_with_startindex_4326(config_sqlite_4326):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_sqlite_4326)
feature_collection = p.query(startindex=20, limit=5, resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 5
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
assert feature['id'] == 'inspireadressen.1744969'
assert 'Egypte' in properties['straatnaam']
geometry = feature.get('geometry', None)
assert geometry is not None
def test_query_bbox_with_startindex_4326(config_sqlite_4326):
"""Testing query for a valid JSON object with geometry"""
p = OGRProvider(config_sqlite_4326)
feature_collection = p.query(
startindex=1, limit=50,
bbox=(5.742, 52.053, 5.773, 52.098),
resulttype='results')
assert feature_collection.get('type', None) == 'FeatureCollection'
features = feature_collection.get('features', None)
assert len(features) == 3
hits = feature_collection.get('numberMatched', None)
assert hits is None
feature = features[0]
properties = feature.get('properties', None)
assert properties is not None
geometry = feature.get('geometry', None)
assert geometry is not None
assert properties['straatnaam'] == 'Egypte'
assert properties['huisnummer'] == '4'