Merge pull request #5 from geopython/misc-fixes

add CSV provider, misc updates

README.md

@@ -9,15 +9,15 @@ cd pygeoapi
 . bin/activate
 git clone https://github.com/geopython/pygeoapi.git
 cd pygeoapi
-pip3 install -r requirements.txt
-pip3 install -r requirements-dev.txt
-pip3 install -e .
+pip install -r requirements.txt
+pip install -r requirements-dev.txt
+pip install -e .
 cp pygeoapi-config.yml local.yml
 vi local.yml
 # update server.url
 # add ES dataset(s) to datasets section
 export PYGEOAPI_CONFIG=`pwd`/local.yml
-python3 pygeoapi/app.py
+python pygeoapi/app.py
 ```
 
 ## Example requests

pygeoapi-config.yml

@@ -24,8 +24,7 @@
 ##############################################################################
 
 server:
-    openapi_def: /path/to/basedir
-    url: http://localhost:5000/wfs
+    url: http://localhost:5000/
     mimetype: application/json; charset=UTF-8
     encoding: utf-8
     language: en-US
@@ -68,11 +67,37 @@ metadata:
         role: pointOfContact
 
 datasets:
+    obs:
+        type: Point
+        title: Observations
+        abstract: Observations
+        keywords:
+            - observations
+            - monitoring
+        crs:
+            - CRS84
+        links:
+            - type: information
+              url: https://github.com/mapserver/mapserver/blob/branch-7-0/msautotest/wxs/data/obs.csv
+            - type: download
+              url: https://raw.githubusercontent.com/mapserver/mapserver/branch-7-0/msautotest/wxs/data/obs.csv
+        extents:
+            spatial:
+                bbox: [-180,-90,180,90]
+            temporal:
+                begin: 2000-10-30T18:24:39Z
+                end: 2007-10-30T08:57:29Z
+        data:
+            type: CSV
+            url: file:///tests/data/obs.csv
+            id_field: id
     landsat-aws:
         type: Polygon
         title: my dataset
         abstract: my dataset
-        keywords: kw1,kw2
+        keywords:
+            - kw1
+            - kw2
         crs:
             - CRS84
         links:
@@ -94,7 +119,8 @@ datasets:
         type: Polygon
         title: Large Lakes
         abstract: lakes of the world, public domain
-        keywords: lakes
+        keywords:
+            - lakes
         crs:
             - CRS84
         links:
@@ -110,5 +136,5 @@ datasets:
                 end: now  # or empty
         data:
             type: GeoJSON
-            url: file://data/ne_110m_lakes.geojson
+            url: file:///tests/data/ne_110m_lakes.geojson
             id_field: null  # null indicates use feature enumeration

pygeoapi/provider/__init__.py

@@ -30,6 +30,7 @@
 import importlib
 
 PROVIDERS = {
+    'CSV': 'pygeoapi.provider.csv.CSVProvider',
     'Elasticsearch': 'pygeoapi.provider.elasticsearch.ElasticsearchProvider',
     'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider'
 }
@@ -39,7 +40,7 @@ def load_provider(provider_obj):
     """
     loads provider by name
 
-    :param name: name of provider
+    :param provider_obj: provider definition dictionary
 
     :returns: provider object
     """

pygeoapi/provider/base.py

@@ -27,6 +27,8 @@
 #
 # =================================================================
 
+from urllib.request import urlparse
+
 
 class BaseProvider(object):
     """generic Provider ABC"""
@@ -35,9 +37,15 @@ class BaseProvider(object):
         """initializer"""
 
         self.type = definition['type']
-        self.url = definition['url']
         self.id_field = definition['id_field']
 
+        parsed_url = urlparse(definition['url'])
+
+        if parsed_url.scheme == 'file':
+            self.url = parsed_url.path
+        else:
+            self.url = definition['url']
+
     def query(self):
         """
         query the provider

pygeoapi/provider/csv.py

@@ -0,0 +1,86 @@
+# =================================================================
+#
+# Authors: Tom Kralidis <tomkralidis@gmail.com>
+#
+# Copyright (c) 2018 Tom Kralidis
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+
+import csv
+from shapely import wkt
+from shapely.geometry import mapping
+
+from pygeoapi.provider.base import BaseProvider
+
+
+class CSVProvider(BaseProvider):
+    """CSV provider"""
+
+    def __init__(self, definition):
+        """initializer"""
+
+        BaseProvider.__init__(self, definition)
+
+        with open(self.url) as ff:
+            self.data = csv.DictReader(ff)
+
+    def _load(self, identifier=None):
+        feature_collection = {
+            'type': 'FeatureCollection',
+            'features': []
+        }
+        with open(self.url) as ff:
+            data = csv.DictReader(ff)
+            for row in data:
+                feature = {'type': 'Feature'}
+                feature['ID'] = row.pop('id')
+                feature['geometry'] = mapping(wkt.loads(row.pop('geom')))
+                feature['properties'] = row
+                if identifier is not None and feature['ID'] == identifier:
+                    return feature
+                feature_collection['features'].append(feature)
+
+        return feature_collection
+
+    def query(self):
+        """
+        CSV query
+
+        :returns: dict of 0..n GeoJSON features
+        """
+
+        return self._load()
+
+    def get(self, identifier):
+        """
+        query CSV id
+
+        :param identifier: feature id
+        :returns: dict of single GeoJSON feature
+        """
+
+        return self._load(identifier)
+
+    def __repr__(self):
+        return '<CSVProvider> {}'.format(self.url)

pygeoapi/provider/elasticsearch.py

@@ -33,7 +33,7 @@ from pygeoapi.provider.base import BaseProvider
 
 
 class ElasticsearchProvider(BaseProvider):
-    """generic Provider ABC"""
+    """Elasticsearch Provider"""
 
     def __init__(self, definition):
         """initializer"""
@@ -50,7 +50,7 @@ class ElasticsearchProvider(BaseProvider):
 
     def query(self):
         """
-        query the provider
+        query ES
 
         :returns: dict of 0..n GeoJSON features
         """
@@ -63,25 +63,29 @@ class ElasticsearchProvider(BaseProvider):
         results = self.es.search(index=self.index_name)
 
         for feature in results['hits']['hits']:
+            id_ = feature['_source']['properties']['identifier']
+            feature['_source']['ID'] = id_
             feature_collection['features'].append(feature['_source'])
 
         return feature_collection
 
     def get(self, identifier):
         """
-        query the provider by id
+        Get ES document by id
 
         :param identifier: feature id
         :returns: dict of single GeoJSON feature
         """
 
         try:
-            es_results = self.es.get(self.index_name, doc_type=self.type_name,
-                                     id=identifier)
+            result = self.es.get(self.index_name, doc_type=self.type_name,
+                                 id=identifier)
+            id_ = result['_source']['properties']['identifier']
+            result['_source']['ID'] = id_
         except Exception as err:
             return None
 
-        return es_results['_source']
+        return result['_source']
 
     def __repr__(self):
         return '<ElasticsearchProvider> {}'.format(self.url)

pygeoapi/provider/geojson.py

@@ -33,8 +33,6 @@ import json
 from pygeoapi.provider.base import BaseProvider
 
 
-
-
 class GeoJSONProvider(BaseProvider):
     """Provider class backed by local GeoJSON files
 
@@ -51,8 +49,7 @@ class GeoJSONProvider(BaseProvider):
         BaseProvider.__init__(self, definition)
 
         # url is a file path, TODO use urlparse or support local paths
-        self.path = self.url.replace("file://", '')
-        self._validate_or_create(self.path)
+        self._validate_or_create(self.url)
 
     @classmethod
     def _validate_or_create(self, path):
@@ -71,7 +68,7 @@ class GeoJSONProvider(BaseProvider):
                 dst.write(json.dumps(empty))
 
     def _load(self):
-        with open(self.path) as src:
+        with open(self.url) as src:
             data = json.loads(src.read())
         return data
 

requirements.txt

@@ -2,5 +2,6 @@ click
 elasticsearch
 flask
 flask_cors
-requests
 pyyaml
+requests
+shapely

tests/data/ne_110m_lakes.geojson

@@ -0,0 +1 @@
+{"type": "FeatureCollection", "features": []}

tests/data/obs.csv

@@ -0,0 +1,6 @@
+id,stn_id,datetime,value,geom
+371,35,"2001-10-30T14:24:55Z",89.9,POINT(-75 45)
+377,35,"2002-10-30T18:31:38Z",93.9,POINT(-75 45)
+238,2147,"2007-10-30T08:57:29Z",103.5,POINT(-79 43)
+297,2147,"2003-10-30T07:37:29Z",93.5,POINT(-79 43)
+964,604,"2000-10-30T18:24:39Z",99.9,POINT(-122 49)