Add csv support via ogr (#410)

Add tests for CSV driver from OGR Add csv example layer to the OGR test configuration
2020-04-16 12:39:08 +02:00
parent c99d1ba1ed
commit c8e535add5
3 changed files with 207 additions and 1 deletions
@@ -623,7 +623,7 @@ class CommonSourceHelper(SourceHelper):

        self.close()

-        sql = "SELECT * FROM {ds_name} LIMIT {limit} OFFSET {offset}".format(
+        sql = 'SELECT * FROM "{ds_name}" LIMIT {limit} OFFSET {offset}'.format(
            ds_name=self.provider.layer_name,
            limit=self.limit,
            offset=self.startindex)
@@ -348,6 +348,48 @@ datasets:

            id_field: objectid

+    cases_italy_per_region_from_github:
+        title: "Cases in Italy - DPC GitHub"
+        description: "Current situation within Italy, number of cases with variation per Italy, provided by ESRI, source data from DPC."
+        keywords: [Daily, Cases Variation, Region]
+        crs:
+            - CRS84
+        links:
+            - type: text/html
+              rel: canonical
+              title: "GitHub DPC repository - COVID-19 raw data for Italy"
+              href: https://github.com/pcm-dpc/COVID-19
+              hreflang: it-IT
+        extents:
+            spatial:
+                bbox: [-180,-90,180,90]
+                crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
+            temporal:
+                begin: 2020-01-01T00:00:00Z
+                end: # or empty
+
+        provider:
+            name: OGR
+            data:
+                source_type: CSV
+                source: /vsicurl/https://raw.githubusercontent.com/pcm-dpc/COVID-19/master/dati-regioni/dpc-covid19-ita-regioni.csv
+                source_srs: EPSG:4326
+                target_srs: EPSG:4326
+                source_capabilities:
+                    paging: True
+                open_options:
+                    X_POSSIBLE_NAMES: long
+                    Y_POSSIBLE_NAMES: lat
+                gdal_ogr_options:
+                    EMPTY_AS_NULL: NO
+                    GDAL_CACHEMAX: 64
+                    # GDAL_HTTP_PROXY: (optional proxy)
+                    # GDAL_PROXY_AUTH: (optional auth for remote WFS)
+                    CPL_DEBUG: NO
+            id_field: fid
+            time_field: data
+            layer: dpc-covid19-ita-regioni
+
 processes:
    hello-world:
        processor:
@@ -0,0 +1,164 @@
+# =================================================================
+#
+# Authors: Francesco Bartoli <xbartolone@gmail.com>
+#
+# Copyright (c) 2020 Francesco Bartoli
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+
+# Needs to be run like: python3 -m pytest
+
+# https://sampleserver6.arcgisonline.com/arcgis/rest/services/CommunityAddressing/FeatureServer/0
+
+import logging
+
+import pytest
+from pygeoapi.provider.ogr import OGRProvider
+
+LOGGER = logging.getLogger(__name__)
+
+
+@pytest.fixture()
+def config_vsicurl_csv():
+    return {
+        'name': 'OGR',
+        'data': {
+            'source_type': 'CSV',
+            'source': '/vsicurl/https://raw.githubusercontent.com/pcm-dpc/COVID-19/master/dati-regioni/dpc-covid19-ita-regioni.csv', # noqa
+            'source_srs': 'EPSG:4326',
+            'target_srs': 'EPSG:4326',
+            'source_capabilities': {
+                'paging': True
+            },
+            'open_options': {
+                'X_POSSIBLE_NAMES': 'long',
+                'Y_POSSIBLE_NAMES': 'lat',
+            },
+            'gdal_ogr_options': {
+                'EMPTY_AS_NULL': 'NO',
+                'GDAL_CACHEMAX': '64',
+                'CPL_DEBUG': 'NO'
+            },
+        },
+        'id_field': 'fid',
+        'time_field': 'data',
+        'layer': 'dpc-covid19-ita-regioni'
+    }
+
+
+def test_get_fields_vsicurl(config_vsicurl_csv):
+    """Testing field types"""
+
+    p = OGRProvider(config_vsicurl_csv)
+    results = p.get_fields()
+    assert results['denominazione_regione'] == 'string'
+    assert results['totale_positivi'] == 'string'
+
+
+def test_get_vsicurl(config_vsicurl_csv):
+    """Testing query for a specific object"""
+
+    p = OGRProvider(config_vsicurl_csv)
+    result = p.get('32')
+    assert result['id'] == 32
+    assert '11' in result['properties']['codice_regione']
+
+
+def test_query_hits_vsicurl(config_vsicurl_csv):
+    """Testing query on entire collection for hits"""
+
+    p = OGRProvider(config_vsicurl_csv)
+    feature_collection = p.query(resulttype='hits')
+    assert feature_collection.get('type', None) == 'FeatureCollection'
+    features = feature_collection.get('features', None)
+    assert len(features) == 0
+    hits = feature_collection.get('numberMatched', None)
+    assert hits is not None
+    assert hits > 100
+
+
+def test_query_bbox_hits_vsicurl(config_vsicurl_csv):
+    """Testing query for a valid JSON object with geometry"""
+
+    p = OGRProvider(config_vsicurl_csv)
+    feature_collection = p.query(
+        bbox=[10.497565, 41.520355, 15.111823, 43.308645],
+        resulttype='hits')
+    assert feature_collection.get('type', None) == 'FeatureCollection'
+    features = feature_collection.get('features', None)
+    assert len(features) == 0
+    hits = feature_collection.get('numberMatched', None)
+    assert hits is not None
+    print('hits={}'.format(hits))
+    assert hits > 1
+
+
+def test_query_with_limit_vsicurl(config_vsicurl_csv):
+    """Testing query for a valid JSON object with geometry"""
+
+    p = OGRProvider(config_vsicurl_csv)
+    feature_collection = p.query(limit=2, resulttype='results')
+    assert feature_collection.get('type', None) == 'FeatureCollection'
+    features = feature_collection.get('features', None)
+    assert len(features) == 2
+    hits = feature_collection.get('numberMatched', None)
+    assert hits is None
+    feature = features[0]
+    properties = feature.get('properties', None)
+    assert properties is not None
+    geometry = feature.get('geometry', None)
+    assert geometry is not None
+
+
+def test_query_with_startindex_vsicurl(config_vsicurl_csv):
+    """Testing query for a valid JSON object with geometry"""
+
+    p = OGRProvider(config_vsicurl_csv)
+    feature_collection = p.query(startindex=20, limit=10, resulttype='results')
+    assert feature_collection.get('type', None) == 'FeatureCollection'
+    features = feature_collection.get('features', None)
+    assert len(features) == 10
+    hits = feature_collection.get('numberMatched', None)
+    assert hits is None
+    feature = features[0]
+    properties = feature.get('properties', None)
+    assert properties is not None
+    assert feature['id'] == 21
+    assert 'Veneto' in properties['denominazione_regione']
+    geometry = feature.get('geometry', None)
+    assert geometry is not None
+
+
+def test_query_with_property_vsicurl(config_vsicurl_csv):
+    """Testing query for a valid JSON object with property filter"""
+
+    p = OGRProvider(config_vsicurl_csv)
+    feature_collection = p.query(
+        startindex=20, limit=10, resulttype='results',
+        properties=[('denominazione_regione', 'Lazio')])
+    assert feature_collection.get('type', None) == 'FeatureCollection'
+    features = feature_collection.get('features', None)
+    assert len(features) == 10
+    for feature in features:
+        assert 'Lazio' in feature['properties']['denominazione_regione']