diff --git a/README.md b/README.md index c2df833..993d961 100644 --- a/README.md +++ b/README.md @@ -9,15 +9,15 @@ cd pygeoapi . bin/activate git clone https://github.com/geopython/pygeoapi.git cd pygeoapi -pip3 install -r requirements.txt -pip3 install -r requirements-dev.txt -pip3 install -e . +pip install -r requirements.txt +pip install -r requirements-dev.txt +pip install -e . cp pygeoapi-config.yml local.yml vi local.yml # update server.url # add ES dataset(s) to datasets section export PYGEOAPI_CONFIG=`pwd`/local.yml -python3 pygeoapi/app.py +python pygeoapi/app.py ``` ## Example requests diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml index dedc325..f45041e 100644 --- a/pygeoapi-config.yml +++ b/pygeoapi-config.yml @@ -24,8 +24,7 @@ ############################################################################## server: - openapi_def: /path/to/basedir - url: http://localhost:5000/wfs + url: http://localhost:5000/ mimetype: application/json; charset=UTF-8 encoding: utf-8 language: en-US @@ -68,11 +67,37 @@ metadata: role: pointOfContact datasets: + obs: + type: Point + title: Observations + abstract: Observations + keywords: + - observations + - monitoring + crs: + - CRS84 + links: + - type: information + url: https://github.com/mapserver/mapserver/blob/branch-7-0/msautotest/wxs/data/obs.csv + - type: download + url: https://raw.githubusercontent.com/mapserver/mapserver/branch-7-0/msautotest/wxs/data/obs.csv + extents: + spatial: + bbox: [-180,-90,180,90] + temporal: + begin: 2000-10-30T18:24:39Z + end: 2007-10-30T08:57:29Z + data: + type: CSV + url: file:///tests/data/obs.csv + id_field: id landsat-aws: type: Polygon title: my dataset abstract: my dataset - keywords: kw1,kw2 + keywords: + - kw1 + - kw2 crs: - CRS84 links: @@ -94,7 +119,8 @@ datasets: type: Polygon title: Large Lakes abstract: lakes of the world, public domain - keywords: lakes + keywords: + - lakes crs: - CRS84 links: @@ -110,5 +136,5 @@ datasets: end: now # or empty data: type: GeoJSON - url: file://data/ne_110m_lakes.geojson + url: file:///tests/data/ne_110m_lakes.geojson id_field: null # null indicates use feature enumeration diff --git a/pygeoapi/provider/__init__.py b/pygeoapi/provider/__init__.py index 8d030a4..ef2b1a5 100644 --- a/pygeoapi/provider/__init__.py +++ b/pygeoapi/provider/__init__.py @@ -30,6 +30,7 @@ import importlib PROVIDERS = { + 'CSV': 'pygeoapi.provider.csv.CSVProvider', 'Elasticsearch': 'pygeoapi.provider.elasticsearch.ElasticsearchProvider', 'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider' } @@ -39,7 +40,7 @@ def load_provider(provider_obj): """ loads provider by name - :param name: name of provider + :param provider_obj: provider definition dictionary :returns: provider object """ diff --git a/pygeoapi/provider/base.py b/pygeoapi/provider/base.py index c4dca0d..42cb15b 100644 --- a/pygeoapi/provider/base.py +++ b/pygeoapi/provider/base.py @@ -27,6 +27,8 @@ # # ================================================================= +from urllib.request import urlparse + class BaseProvider(object): """generic Provider ABC""" @@ -35,9 +37,15 @@ class BaseProvider(object): """initializer""" self.type = definition['type'] - self.url = definition['url'] self.id_field = definition['id_field'] + parsed_url = urlparse(definition['url']) + + if parsed_url.scheme == 'file': + self.url = parsed_url.path + else: + self.url = definition['url'] + def query(self): """ query the provider diff --git a/pygeoapi/provider/csv.py b/pygeoapi/provider/csv.py new file mode 100644 index 0000000..4a67797 --- /dev/null +++ b/pygeoapi/provider/csv.py @@ -0,0 +1,86 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2018 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import csv +from shapely import wkt +from shapely.geometry import mapping + +from pygeoapi.provider.base import BaseProvider + + +class CSVProvider(BaseProvider): + """CSV provider""" + + def __init__(self, definition): + """initializer""" + + BaseProvider.__init__(self, definition) + + with open(self.url) as ff: + self.data = csv.DictReader(ff) + + def _load(self, identifier=None): + feature_collection = { + 'type': 'FeatureCollection', + 'features': [] + } + with open(self.url) as ff: + data = csv.DictReader(ff) + for row in data: + feature = {'type': 'Feature'} + feature['ID'] = row.pop('id') + feature['geometry'] = mapping(wkt.loads(row.pop('geom'))) + feature['properties'] = row + if identifier is not None and feature['ID'] == identifier: + return feature + feature_collection['features'].append(feature) + + return feature_collection + + def query(self): + """ + CSV query + + :returns: dict of 0..n GeoJSON features + """ + + return self._load() + + def get(self, identifier): + """ + query CSV id + + :param identifier: feature id + :returns: dict of single GeoJSON feature + """ + + return self._load(identifier) + + def __repr__(self): + return ' {}'.format(self.url) diff --git a/pygeoapi/provider/elasticsearch.py b/pygeoapi/provider/elasticsearch.py index 3c7fbe8..2a5b086 100644 --- a/pygeoapi/provider/elasticsearch.py +++ b/pygeoapi/provider/elasticsearch.py @@ -33,7 +33,7 @@ from pygeoapi.provider.base import BaseProvider class ElasticsearchProvider(BaseProvider): - """generic Provider ABC""" + """Elasticsearch Provider""" def __init__(self, definition): """initializer""" @@ -50,7 +50,7 @@ class ElasticsearchProvider(BaseProvider): def query(self): """ - query the provider + query ES :returns: dict of 0..n GeoJSON features """ @@ -63,25 +63,29 @@ class ElasticsearchProvider(BaseProvider): results = self.es.search(index=self.index_name) for feature in results['hits']['hits']: + id_ = feature['_source']['properties']['identifier'] + feature['_source']['ID'] = id_ feature_collection['features'].append(feature['_source']) return feature_collection def get(self, identifier): """ - query the provider by id + Get ES document by id :param identifier: feature id :returns: dict of single GeoJSON feature """ try: - es_results = self.es.get(self.index_name, doc_type=self.type_name, - id=identifier) + result = self.es.get(self.index_name, doc_type=self.type_name, + id=identifier) + id_ = result['_source']['properties']['identifier'] + result['_source']['ID'] = id_ except Exception as err: return None - return es_results['_source'] + return result['_source'] def __repr__(self): return ' {}'.format(self.url) diff --git a/pygeoapi/provider/geojson.py b/pygeoapi/provider/geojson.py index 9bc05f2..abff1bc 100644 --- a/pygeoapi/provider/geojson.py +++ b/pygeoapi/provider/geojson.py @@ -33,8 +33,6 @@ import json from pygeoapi.provider.base import BaseProvider - - class GeoJSONProvider(BaseProvider): """Provider class backed by local GeoJSON files @@ -51,8 +49,7 @@ class GeoJSONProvider(BaseProvider): BaseProvider.__init__(self, definition) # url is a file path, TODO use urlparse or support local paths - self.path = self.url.replace("file://", '') - self._validate_or_create(self.path) + self._validate_or_create(self.url) @classmethod def _validate_or_create(self, path): @@ -71,7 +68,7 @@ class GeoJSONProvider(BaseProvider): dst.write(json.dumps(empty)) def _load(self): - with open(self.path) as src: + with open(self.url) as src: data = json.loads(src.read()) return data diff --git a/requirements.txt b/requirements.txt index 8a28bd9..9896e14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ click elasticsearch flask flask_cors -requests pyyaml +requests +shapely diff --git a/tests/data/ne_110m_lakes.geojson b/tests/data/ne_110m_lakes.geojson new file mode 100644 index 0000000..e2c5f1b --- /dev/null +++ b/tests/data/ne_110m_lakes.geojson @@ -0,0 +1 @@ +{"type": "FeatureCollection", "features": []} diff --git a/tests/data/obs.csv b/tests/data/obs.csv new file mode 100644 index 0000000..f173d9c --- /dev/null +++ b/tests/data/obs.csv @@ -0,0 +1,6 @@ +id,stn_id,datetime,value,geom +371,35,"2001-10-30T14:24:55Z",89.9,POINT(-75 45) +377,35,"2002-10-30T18:31:38Z",93.9,POINT(-75 45) +238,2147,"2007-10-30T08:57:29Z",103.5,POINT(-79 43) +297,2147,"2003-10-30T07:37:29Z",93.5,POINT(-79 43) +964,604,"2000-10-30T18:24:39Z",99.9,POINT(-122 49)