diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml index a2be7fc..3cf6770 100644 --- a/pygeoapi-config.yml +++ b/pygeoapi-config.yml @@ -273,6 +273,47 @@ datasets: id_field: gml_id layer: rdinfo:stations + ogr_addresses_sqlite: + # Gotcha: don't use the name 'ogc_fid' as id-field in your SQLite tables: + # OGR will not show this field within GeoJSON properties dict. + title: Dutch addresses (subset Otterlo). OGR SQLite Driver + description: Dutch addresses subset. + keywords: + - Netherlands + - addresses + - INSPIRE + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: information + href: http://www.nationaalgeoregister.nl/geonetwork/srv/dut/catalog.search#/metadata/4074b3c3-ca85-45ad-bc0d-b5fca8540z0b + hreflang: nl-NL + extents: + spatial: + bbox: [50.7539, 7.21097, 53.4658, 3.37087] + temporal: + begin: None + end: now # or empty + provider: + name: OGR + data: + source_type: SQLite + source: tests/data/dutch_addresses_4326.sqlite + source_srs: EPSG:4326 + target_srs: EPSG:4326 + source_capabilities: + paging: True + + gdal_ogr_options: + EMPTY_AS_NULL: NO + GDAL_CACHEMAX: 64 + CPL_DEBUG: NO + + id_field: id + layer: ogrgeojson + processes: hello-world: processor: diff --git a/pygeoapi/provider/ogr.py b/pygeoapi/provider/ogr.py index 2ca5b5b..a455398 100644 --- a/pygeoapi/provider/ogr.py +++ b/pygeoapi/provider/ogr.py @@ -40,14 +40,25 @@ LOGGER = logging.getLogger(__name__) class OGRProvider(BaseProvider): - """OGR Provider""" + """ + OGR Provider. Uses GDAL/OGR Python-bindings to access OGR + Vector sources. References: + https://pcjericks.github.io/py-gdalogr-cookbook/ + https://www.gdal.org/ogr_formats.html (per-driver specifics). + + In theory any OGR source type (Driver) could be used, although + some Source Types are Driver-specific handling. This is handled + in Source Helper classes, instantiated per Source-Type. + + The following Source Types have been tested to work: + GeoPackage (GPKG), SQLite, GeoJSON, ESRI Shapefile, WFS v2. + """ # To deal with some OGR Source-Driver specifics. SOURCE_HELPERS = { - 'WFS': 'pygeoapi.provider.ogr.WFSHelper', - 'ESRI Shapefile': 'pygeoapi.provider.ogr.ShapefileHelper', 'ESRIJSON': 'pygeoapi.provider.ogr.ESRIJSONHelper', - 'GPKG': 'pygeoapi.provider.ogr.GPKGHelper' + 'WFS': 'pygeoapi.provider.ogr.WFSHelper', + '*': 'pygeoapi.provider.ogr.CommonSourceHelper' } def __init__(self, provider_def): @@ -152,10 +163,16 @@ class OGRProvider(BaseProvider): self._load_source_helper(self.data_def['source_type']) - # Init + # Layer name is required + self.layer_name = provider_def.get('layer', None) + if not self.layer_name: + msg = 'Need explicit \'layer\' attr in provider config' + LOGGER.error(msg) + raise Exception(msg) + + # Init driver and Source connection self.driver = None self.conn = None - self.layer_name = provider_def.get('layer', None) def _open(self): source_type = self.data_def['source_type'] @@ -176,6 +193,7 @@ class OGRProvider(BaseProvider): self.source_helper.disable_paging() def _close(self): + self.source_helper.close() self.conn = None LOGGER.debug('closed self.conn') @@ -185,18 +203,8 @@ class OGRProvider(BaseProvider): if not self.conn: self._open() - if not self.layer_name: - # E.g. Shapefiles may not have explicitly named Layers - layer = self.conn.GetLayer(0) - else: - layer = self.conn.GetLayerByName(self.layer_name) - - if not layer: - msg = 'Cannot get Layer {} from OGR Source'.format(self.layer_name) - LOGGER.error(msg) - raise Exception(msg) - - return layer + # Delegate getting Layer to SourceHelper + return self.source_helper.get_layer() def get_fields(self): """ @@ -265,6 +273,8 @@ class OGRProvider(BaseProvider): # layer.SetSpatialFilterRect( # float(minx), float(miny), float(maxx), float(maxy)) + + # Make response based on resulttype specified if resulttype == 'hits': LOGGER.debug('hits only specified') result = self._response_feature_hits(layer) @@ -319,14 +329,12 @@ class OGRProvider(BaseProvider): :returns: Source Helper object """ - + helper_type = source_type if source_type not in OGRProvider.SOURCE_HELPERS.keys(): - msg = 'No Helper found for OGR Source type: {}'.format(source_type) - LOGGER.exception(msg) - raise InvalidHelperError(msg) + helper_type = '*' # Create object from full package.class name string. - source_helper_class = OGRProvider.SOURCE_HELPERS[source_type] + source_helper_class = OGRProvider.SOURCE_HELPERS[helper_type] packagename, classname = source_helper_class.rsplit('.', 1) module = importlib.import_module(packagename) @@ -340,7 +348,11 @@ class OGRProvider(BaseProvider): geom.Transform(self.transform_out) json_feature = ogr_feature.ExportToJson(as_object=True) - json_feature['id'] = json_feature['properties'].pop(self.id_field) + try: + json_feature['id'] = json_feature['properties'].pop(self.id_field) + except Exception: + json_feature['id'] = ogr_feature.GetFID() + return json_feature def _response_feature_collection(self, layer, limit): @@ -397,9 +409,15 @@ class InvalidHelperError(Exception): class SourceHelper: + """ + Helper classes for OGR-specific Source Types (Drivers). + For some actions Driver-specific settings or processing is + required. This is delegated to the OGR SourceHelper classes. + """ + def __init__(self, provider): """ - Initialize object + Initialize object with related OGRProvider object. :param provider: provider instance @@ -407,6 +425,31 @@ class SourceHelper: """ self.provider = provider + def close(self): + """ + OGR Driver-specific handling of closing dataset. + Default is no specific handling. + + """ + + pass + + def get_layer(self): + """ + Default action to get a Layer object from opened OGR Driver. + :return: + """ + + layer = self.provider.conn.GetLayerByName(self.provider.layer_name) + + if not layer: + msg = 'Cannot get Layer {} from OGR Source'.\ + format(self.provider.layer_name) + LOGGER.error(msg) + raise Exception(msg) + + return layer + def enable_paging(self, startindex=-1, limit=-1): """ Enable paged access to dataset (OGR Driver-specific) @@ -423,8 +466,11 @@ class SourceHelper: pass -class GPKGHelper(SourceHelper): - +class CommonSourceHelper(SourceHelper): + """ + SourceHelper for most common OGR Source types: + Shapefile, GeoPackage, SQLite, GeoJSON etc. + """ def __init__(self, provider): """ Initialize object @@ -433,22 +479,80 @@ class GPKGHelper(SourceHelper): :returns: pygeoapi.providers.ogr.SourceHelper """ - self.provider = provider SourceHelper.__init__(self, provider) + self.startindex = -1 + self.limit = -1 + self.result_set = None - -class ShapefileHelper(SourceHelper): - - def __init__(self, provider): + def close(self): """ - Initialize object + OGR Driver-specific handling of closing dataset. + If ExecuteSQL has been (successfully) called + must close ResultSet explicitly. + https://gis.stackexchange.com/questions/114112/ + explicitly-close-a-ogr-result-object-from-a-call-to-executesql - :param provider: provider instance - - :returns: pygeoapi.providers.ogr.SourceHelper """ - self.provider = provider - SourceHelper.__init__(self, provider) + + if not self.result_set: + return + + try: + self.provider.conn.ReleaseResultSet(self.result_set) + except Exception as err: + msg = 'ReleaseResultSet exception for Layer {}'.format( + self.provider.layer_name) + LOGGER.error(msg, err) + finally: + self.result_set = None + + def enable_paging(self, startindex=-1, limit=-1): + """ + Enable paged access to dataset (OGR Driver-specific) + using OGR SQL https://www.gdal.org/ogr_sql.html + e.g. SELECT * FROM poly LIMIT 10 OFFSET 30 + + """ + self.startindex = startindex + self.limit = limit + + def disable_paging(self): + """ + Disable paged access to dataset (OGR Driver-specific) + """ + + pass + + def get_layer(self): + """ + Gets OGR Layer from opened OGR dataset. + When startindex defined 1 or greater will invoke + OGR SQL SELECT with LIMIT and OFFSET and return + as Layer as ResultSet from ExecuteSQL on dataset. + :return: OGR layer object + """ + if self.startindex <= 0: + return SourceHelper.get_layer(self) + + self.close() + + sql = "SELECT * FROM {ds_name} LIMIT {limit} OFFSET {offset}".format( + ds_name=self.provider.layer_name, + limit=self.limit, + offset=self.startindex) + self.result_set = self.provider.conn.ExecuteSQL(sql) + + # Reset since needs to be set each time explicitly + self.startindex = -1 + self.limit = -1 + + if not self.result_set: + msg = 'Cannot get Layer {} via ExecuteSQL'.format( + self.provider.layer_name) + LOGGER.error(msg) + raise Exception(msg) + + return self.result_set class ESRIJSONHelper(SourceHelper): @@ -461,7 +565,6 @@ class ESRIJSONHelper(SourceHelper): :returns: pygeoapi.providers.ogr.SourceHelper """ - self.provider = provider SourceHelper.__init__(self, provider) def enable_paging(self, startindex=-1, limit=-1): @@ -501,7 +604,6 @@ class WFSHelper(SourceHelper): :returns: pygeoapi.providers.ogr.SourceHelper """ - self.provider = provider SourceHelper.__init__(self, provider) def enable_paging(self, startindex=-1, limit=-1): diff --git a/requirements-dev.txt b/requirements-dev.txt index 5362e39..6cf629e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,4 +15,4 @@ pytest-env pyOpenSSL==17.5.0 ndg-httpsclient==0.4.4 pyasn1==0.4.2 -GDAL>=2.1 +GDAL>=2.2 diff --git a/tests/data/dutch_addresses_4326.sqlite b/tests/data/dutch_addresses_4326.sqlite new file mode 100644 index 0000000..a2a9340 Binary files /dev/null and b/tests/data/dutch_addresses_4326.sqlite differ diff --git a/tests/test_ogr_gpkg_provider.py b/tests/test_ogr_gpkg_provider.py index e26b710..c03191d 100644 --- a/tests/test_ogr_gpkg_provider.py +++ b/tests/test_ogr_gpkg_provider.py @@ -67,7 +67,8 @@ def config_gpkg_4326(): 'paging': True }, }, - 'id_field': 'id' + 'id_field': 'id', + 'layer': 'OGRGeoJSON' } @@ -86,7 +87,8 @@ def config_gpkg_28992(): 'paging': True }, }, - 'id_field': 'id' + 'id_field': 'id', + 'layer': 'OGRGeoJSON' } @@ -249,3 +251,85 @@ def test_query_with_limit_4326(config_gpkg_4326): assert properties is not None geometry = feature.get('geometry', None) assert geometry is not None + + +def test_query_with_startindex_28992(config_gpkg_28992): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_gpkg_28992) + feature_collection = p.query(startindex=20, limit=5, resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 5 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + assert feature['id'] == 'inspireadressen.1744969' + assert 'Egypte' in properties['straatnaam'] + geometry = feature.get('geometry', None) + assert geometry is not None + + +def test_query_with_startindex_4326(config_gpkg_4326): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_gpkg_4326) + feature_collection = p.query(startindex=20, limit=5, resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 5 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + assert feature['id'] == 'inspireadressen.1744969' + assert 'Egypte' in properties['straatnaam'] + geometry = feature.get('geometry', None) + assert geometry is not None + + +def test_query_bbox_with_startindex_28992(config_gpkg_28992): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_gpkg_28992) + feature_collection = p.query( + startindex=10, limit=5, + bbox=(5.742, 52.053, 5.773, 52.098), + resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 5 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + geometry = feature.get('geometry', None) + assert geometry is not None + assert properties['straatnaam'] == 'Buurtweg' + assert properties['huisnummer'] == '4' + + +def test_query_bbox_with_startindex_4326(config_gpkg_4326): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_gpkg_4326) + feature_collection = p.query( + startindex=1, limit=5, + bbox=(5.742, 52.053, 5.773, 52.098), + resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 5 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + geometry = feature.get('geometry', None) + assert geometry is not None + assert properties['straatnaam'] == 'Egypte' + assert properties['huisnummer'] == '6' diff --git a/tests/test_ogr_shapefile_provider.py b/tests/test_ogr_shapefile_provider.py index 50cb7af..90793e6 100644 --- a/tests/test_ogr_shapefile_provider.py +++ b/tests/test_ogr_shapefile_provider.py @@ -25,7 +25,8 @@ def config_shapefile_4326(): 'paging': True }, }, - 'id_field': 'id' + 'id_field': 'id', + 'layer': 'inspireadressen' } @@ -44,7 +45,8 @@ def config_shapefile_28992(): 'paging': True }, }, - 'id_field': 'id' + 'id_field': 'id', + 'layer': 'inspireadressen' } @@ -207,3 +209,85 @@ def test_query_with_limit_4326(config_shapefile_4326): assert properties is not None geometry = feature.get('geometry', None) assert geometry is not None + + +def test_query_with_startindex_28992(config_shapefile_28992): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_shapefile_28992) + feature_collection = p.query(startindex=20, limit=5, resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 5 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + assert feature['id'] == 'inspireadressen.1744969' + assert 'Egypte' in properties['straatnaam'] + geometry = feature.get('geometry', None) + assert geometry is not None + + +def test_query_with_startindex_4326(config_shapefile_4326): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_shapefile_4326) + feature_collection = p.query(startindex=20, limit=5, resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 5 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + assert feature['id'] == 'inspireadressen.1744969' + assert 'Egypte' in properties['straatnaam'] + geometry = feature.get('geometry', None) + assert geometry is not None + + +def test_query_bbox_with_startindex_28992(config_shapefile_28992): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_shapefile_28992) + feature_collection = p.query( + startindex=10, limit=5, + bbox=(5.742, 52.053, 5.773, 52.098), + resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 5 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + geometry = feature.get('geometry', None) + assert geometry is not None + assert properties['straatnaam'] == 'Buurtweg' + assert properties['huisnummer'] == '4' + + +def test_query_bbox_with_startindex_4326(config_shapefile_4326): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_shapefile_4326) + feature_collection = p.query( + startindex=1, limit=5, + bbox=(5.742, 52.053, 5.773, 52.098), + resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 5 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + geometry = feature.get('geometry', None) + assert geometry is not None + assert properties['straatnaam'] == 'Egypte' + assert properties['huisnummer'] == '6' diff --git a/tests/test_ogr_sqlite_provider.py b/tests/test_ogr_sqlite_provider.py new file mode 100644 index 0000000..00570b3 --- /dev/null +++ b/tests/test_ogr_sqlite_provider.py @@ -0,0 +1,156 @@ +# Needs to be run like: python3 -m pytest + +import logging + +import pytest + +from pygeoapi.provider.ogr import OGRProvider + +LOGGER = logging.getLogger(__name__) + + +# Testing with SQLite files with identical features +# (all 2481 addresses in Otterlo Netherlands). + +@pytest.fixture() +def config_sqlite_4326(): + return { + 'name': 'OGR', + 'data': { + 'source_type': 'SQLite', + 'source': + './tests/data/dutch_addresses_4326.sqlite', + 'source_srs': 'EPSG:4326', + 'target_srs': 'EPSG:4326', + 'source_capabilities': { + 'paging': True + }, + }, + 'id_field': 'id', + 'layer': 'ogrgeojson' + } + + +def test_get_fields_4326(config_sqlite_4326): + """Testing field types""" + p = OGRProvider(config_sqlite_4326) + results = p.get_fields() + assert results['straatnaam'] == 'string' + assert results['huisnummer'] == 'string' + + +def test_get_4326(config_sqlite_4326): + """Testing query for a specific object""" + p = OGRProvider(config_sqlite_4326) + result = p.get('inspireadressen.1747652') + assert result['id'] == 'inspireadressen.1747652' + assert 'Mosselsepad' in result['properties']['straatnaam'] + + +def test_query_hits_4326(config_sqlite_4326): + """Testing query on entire collection for hits""" + + p = OGRProvider(config_sqlite_4326) + feature_collection = p.query(resulttype='hits') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) is 0 + hits = feature_collection.get('numberMatched', None) + assert hits is not None + assert hits == 2481 + + +def test_query_bbox_hits_4326(config_sqlite_4326): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_sqlite_4326) + # feature_collection = p.query( + # bbox=[120000, 480000, 124000, 487000], resulttype='hits') + feature_collection = p.query( + bbox=[5.763409, 52.060197, 5.769256, 52.061976], resulttype='hits') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) is 0 + hits = feature_collection.get('numberMatched', None) + assert hits is not None + print('hits={}'.format(hits)) + assert hits is 1 + + +def test_query_bbox_4326(config_sqlite_4326): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_sqlite_4326) + # feature_collection = p.query( + # bbox=[180800, 452500, 181200, 452700], resulttype='results') + feature_collection = p.query( + bbox=(5.763409, 52.060197, 5.769256, 52.061976), resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 1 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + geometry = feature.get('geometry', None) + assert geometry is not None + assert properties['straatnaam'] == 'Planken Wambuisweg' + + +def test_query_with_limit_4326(config_sqlite_4326): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_sqlite_4326) + feature_collection = p.query(limit=5, resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 5 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + geometry = feature.get('geometry', None) + assert geometry is not None + + +def test_query_with_startindex_4326(config_sqlite_4326): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_sqlite_4326) + feature_collection = p.query(startindex=20, limit=5, resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 5 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + assert feature['id'] == 'inspireadressen.1744969' + assert 'Egypte' in properties['straatnaam'] + geometry = feature.get('geometry', None) + assert geometry is not None + + +def test_query_bbox_with_startindex_4326(config_sqlite_4326): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_sqlite_4326) + feature_collection = p.query( + startindex=1, limit=50, + bbox=(5.742, 52.053, 5.773, 52.098), + resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 3 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + geometry = feature.get('geometry', None) + assert geometry is not None + assert properties['straatnaam'] == 'Egypte' + assert properties['huisnummer'] == '4'