From c8e535add56a178bd30c2777db616c0c2d3dc5df Mon Sep 17 00:00:00 2001 From: Francesco Bartoli Date: Thu, 16 Apr 2020 12:39:08 +0200 Subject: [PATCH] Add csv support via ogr (#410) Add tests for CSV driver from OGR Add csv example layer to the OGR test configuration --- pygeoapi/provider/ogr.py | 2 +- tests/pygeoapi-test-ogr-config.yml | 42 ++++++++ tests/test_ogr_csv_provider.py | 164 +++++++++++++++++++++++++++++ 3 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 tests/test_ogr_csv_provider.py diff --git a/pygeoapi/provider/ogr.py b/pygeoapi/provider/ogr.py index d38df62..e64fd33 100644 --- a/pygeoapi/provider/ogr.py +++ b/pygeoapi/provider/ogr.py @@ -623,7 +623,7 @@ class CommonSourceHelper(SourceHelper): self.close() - sql = "SELECT * FROM {ds_name} LIMIT {limit} OFFSET {offset}".format( + sql = 'SELECT * FROM "{ds_name}" LIMIT {limit} OFFSET {offset}'.format( ds_name=self.provider.layer_name, limit=self.limit, offset=self.startindex) diff --git a/tests/pygeoapi-test-ogr-config.yml b/tests/pygeoapi-test-ogr-config.yml index 653fcc6..abd0d6c 100644 --- a/tests/pygeoapi-test-ogr-config.yml +++ b/tests/pygeoapi-test-ogr-config.yml @@ -348,6 +348,48 @@ datasets: id_field: objectid + cases_italy_per_region_from_github: + title: "Cases in Italy - DPC GitHub" + description: "Current situation within Italy, number of cases with variation per Italy, provided by ESRI, source data from DPC." + keywords: [Daily, Cases Variation, Region] + crs: + - CRS84 + links: + - type: text/html + rel: canonical + title: "GitHub DPC repository - COVID-19 raw data for Italy" + href: https://github.com/pcm-dpc/COVID-19 + hreflang: it-IT + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2020-01-01T00:00:00Z + end: # or empty + + provider: + name: OGR + data: + source_type: CSV + source: /vsicurl/https://raw.githubusercontent.com/pcm-dpc/COVID-19/master/dati-regioni/dpc-covid19-ita-regioni.csv + source_srs: EPSG:4326 + target_srs: EPSG:4326 + source_capabilities: + paging: True + open_options: + X_POSSIBLE_NAMES: long + Y_POSSIBLE_NAMES: lat + gdal_ogr_options: + EMPTY_AS_NULL: NO + GDAL_CACHEMAX: 64 + # GDAL_HTTP_PROXY: (optional proxy) + # GDAL_PROXY_AUTH: (optional auth for remote WFS) + CPL_DEBUG: NO + id_field: fid + time_field: data + layer: dpc-covid19-ita-regioni + processes: hello-world: processor: diff --git a/tests/test_ogr_csv_provider.py b/tests/test_ogr_csv_provider.py new file mode 100644 index 0000000..8fa2d51 --- /dev/null +++ b/tests/test_ogr_csv_provider.py @@ -0,0 +1,164 @@ +# ================================================================= +# +# Authors: Francesco Bartoli +# +# Copyright (c) 2020 Francesco Bartoli +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +# Needs to be run like: python3 -m pytest + +# https://sampleserver6.arcgisonline.com/arcgis/rest/services/CommunityAddressing/FeatureServer/0 + +import logging + +import pytest +from pygeoapi.provider.ogr import OGRProvider + +LOGGER = logging.getLogger(__name__) + + +@pytest.fixture() +def config_vsicurl_csv(): + return { + 'name': 'OGR', + 'data': { + 'source_type': 'CSV', + 'source': '/vsicurl/https://raw.githubusercontent.com/pcm-dpc/COVID-19/master/dati-regioni/dpc-covid19-ita-regioni.csv', # noqa + 'source_srs': 'EPSG:4326', + 'target_srs': 'EPSG:4326', + 'source_capabilities': { + 'paging': True + }, + 'open_options': { + 'X_POSSIBLE_NAMES': 'long', + 'Y_POSSIBLE_NAMES': 'lat', + }, + 'gdal_ogr_options': { + 'EMPTY_AS_NULL': 'NO', + 'GDAL_CACHEMAX': '64', + 'CPL_DEBUG': 'NO' + }, + }, + 'id_field': 'fid', + 'time_field': 'data', + 'layer': 'dpc-covid19-ita-regioni' + } + + +def test_get_fields_vsicurl(config_vsicurl_csv): + """Testing field types""" + + p = OGRProvider(config_vsicurl_csv) + results = p.get_fields() + assert results['denominazione_regione'] == 'string' + assert results['totale_positivi'] == 'string' + + +def test_get_vsicurl(config_vsicurl_csv): + """Testing query for a specific object""" + + p = OGRProvider(config_vsicurl_csv) + result = p.get('32') + assert result['id'] == 32 + assert '11' in result['properties']['codice_regione'] + + +def test_query_hits_vsicurl(config_vsicurl_csv): + """Testing query on entire collection for hits""" + + p = OGRProvider(config_vsicurl_csv) + feature_collection = p.query(resulttype='hits') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 0 + hits = feature_collection.get('numberMatched', None) + assert hits is not None + assert hits > 100 + + +def test_query_bbox_hits_vsicurl(config_vsicurl_csv): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_vsicurl_csv) + feature_collection = p.query( + bbox=[10.497565, 41.520355, 15.111823, 43.308645], + resulttype='hits') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 0 + hits = feature_collection.get('numberMatched', None) + assert hits is not None + print('hits={}'.format(hits)) + assert hits > 1 + + +def test_query_with_limit_vsicurl(config_vsicurl_csv): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_vsicurl_csv) + feature_collection = p.query(limit=2, resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 2 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + geometry = feature.get('geometry', None) + assert geometry is not None + + +def test_query_with_startindex_vsicurl(config_vsicurl_csv): + """Testing query for a valid JSON object with geometry""" + + p = OGRProvider(config_vsicurl_csv) + feature_collection = p.query(startindex=20, limit=10, resulttype='results') + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 10 + hits = feature_collection.get('numberMatched', None) + assert hits is None + feature = features[0] + properties = feature.get('properties', None) + assert properties is not None + assert feature['id'] == 21 + assert 'Veneto' in properties['denominazione_regione'] + geometry = feature.get('geometry', None) + assert geometry is not None + + +def test_query_with_property_vsicurl(config_vsicurl_csv): + """Testing query for a valid JSON object with property filter""" + + p = OGRProvider(config_vsicurl_csv) + feature_collection = p.query( + startindex=20, limit=10, resulttype='results', + properties=[('denominazione_regione', 'Lazio')]) + assert feature_collection.get('type', None) == 'FeatureCollection' + features = feature_collection.get('features', None) + assert len(features) == 10 + for feature in features: + assert 'Lazio' in feature['properties']['denominazione_regione']