diff --git a/.gitignore b/.gitignore index 7bbc71c..a9a8351 100644 --- a/.gitignore +++ b/.gitignore @@ -99,3 +99,7 @@ ENV/ # mypy .mypy_cache/ + +# pygeoapi artifacts +local.config.yml +local.swagger.yml diff --git a/README.md b/README.md index a1523e1..f177364 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,15 @@ pip3 install -r requirements-dev.txt pip3 install -e . cp openapi/wfs/0.0.1/pygeoapi-openapi.yml local.swagger.yml cp pygeoapi-config.yml local.config.yml -python flask_app.py - +vi local.config.yml +# TODO: what is most important to edit? +vi local.swagger.yml +# TODO: what is most important to edit? +export PYGEOAPI_CONFIG=/path/to/local.config.yml +export PYGEOAPI_SWAGGER=/path/to/local.swagger.yml +pygeoapi serve ``` -Edit `local.config.yml` and `local.swagger.yml` - ## Example requests Try the swagger ui at `http://localhost:5000/ui` @@ -30,7 +33,10 @@ or ```bash # feature collection metadata curl http://localhost:5000/ -curl http://localhost:5000/api +# conformance +curl http://localhost:5000/api/conformance +# feature collection curl http://localhost:5000/obs +# feature curl http://localhost:5000/obs/371 ``` diff --git a/openapi/wfs/0.0.1/pygeoapi-openapi.yml b/openapi/wfs/0.0.1/pygeoapi-openapi.yml index 33941d5..2c450a8 100644 --- a/openapi/wfs/0.0.1/pygeoapi-openapi.yml +++ b/openapi/wfs/0.0.1/pygeoapi-openapi.yml @@ -22,6 +22,7 @@ # OR OTHER DEALINGS IN THE SOFTWARE. # ############################################################################## + swagger: '2.0' info: title: A sample API conforming to the OGC Web Feature Service standard @@ -93,11 +94,11 @@ paths: summary: retrieve observation features description: >- Sample observations - operationId: pygeoapi.views.getFeatures + operationId: pygeoapi.views.get_features tags: - Features parameters: - - name: ds # constant parameter + - name: dataset # constant parameter in: query type: string enum: @@ -125,11 +126,11 @@ paths: '/obs/{id}': get: summary: retrieve a feature - operationId: pygeoapi.views.getFeature + operationId: pygeoapi.views.get_feature tags: - Features parameters: - - name: ds # constant parameter + - name: dataset # constant parameter in: query type: string enum: diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml index c925d39..eb235db 100644 --- a/pygeoapi-config.yml +++ b/pygeoapi-config.yml @@ -1,6 +1,47 @@ server: host: localhost port: 5000 + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + language: en-US + cors: true + pretty_print: true + limit: 10 + +logging: + level: INFO + #logfile: /tmp/pygeoapi.log + +metadata: + identification: + title: pygeoapi + abstract: pygeoapi provides an API to geospatial data + keywords: + - geospatial + - data + - api + keywords_type: theme + fees: None + accessconstraints: None + provider: + name: Organization Name + url: https://github.com/geopython/pygeoapi + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Hours of Service + instructions: During hours of service. Off on weekends. + role: pointOfContact + datasets: obs: type: Point @@ -24,5 +65,52 @@ datasets: end: 2007-10-30T08:57:29Z data: type: CSV - url: ./tests/data/obs.csv + url: file:///tests/data/obs.csv id_field: id + landsat-aws: + type: Polygon + title: my dataset + abstract: my dataset + keywords: + - kw1 + - kw2 + crs: + - CRS84 + links: + - type: information + url: http://example.org/dataset/index.html + - type: download + url: http://example.org/dataset/data.tgz + extents: + spatial: + bbox: [-180,-90,180,90] + temporal: + begin: 2011-11-11 + end: now # or empty + data: + type: Elasticsearch + url: http://localhost:9200/index/type + id_field: id_ + lakes: + type: Polygon + title: Large Lakes + abstract: lakes of the world, public domain + keywords: + - lakes + crs: + - CRS84 + links: + - type: information + url: http://www.naturalearthdata.com/ + - type: download + url: http://www.naturalearthdata.com/ + extents: + spatial: + bbox: [-180,-90,180,90] + temporal: + begin: 2011-11-11 + end: now # or empty + data: + type: GeoJSON + url: file:///tests/data/ne_110m_lakes.geojson + id_field: null # null indicates use feature enumeration diff --git a/pygeoapi/__init__.py b/pygeoapi/__init__.py index 47afb55..7e7a38d 100644 --- a/pygeoapi/__init__.py +++ b/pygeoapi/__init__.py @@ -29,6 +29,8 @@ import click +from pygeoapi.flask_app import serve + __version__ = '0.1.dev0' @@ -36,3 +38,6 @@ __version__ = '0.1.dev0' @click.version_option(version=__version__) def cli(): pass + + +cli.add_command(serve) diff --git a/pygeoapi/config.py b/pygeoapi/config.py index 4e9a4ab..b4bbe6d 100644 --- a/pygeoapi/config.py +++ b/pygeoapi/config.py @@ -1,4 +1,38 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Norman Barker +# +# Copyright (c) 2018 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import os + import yaml -with open('local.config.yml', 'r') as ymlfile: - settings = yaml.load(ymlfile) +with open(os.environ.get('PYGEOAPI_CONFIG')) as ff: + settings = yaml.load(ff) + +settings['swagger'] = os.environ.get('PYGEOAPI_SWAGGER') diff --git a/flask_app.py b/pygeoapi/flask_app.py similarity index 61% rename from flask_app.py rename to pygeoapi/flask_app.py index ac3a9f7..8ace2b2 100644 --- a/flask_app.py +++ b/pygeoapi/flask_app.py @@ -1,7 +1,7 @@ # ================================================================= # # Authors: Tom Kralidis -# : Norman Barker +# Norman Barker # # Copyright (c) 2018 Tom Kralidis # @@ -28,12 +28,38 @@ # # ================================================================= +import click import connexion + +from pygeoapi.log import setup_logger from pygeoapi.config import settings -if __name__ == '__main__': - app = connexion.FlaskApp(__name__, port=int(settings['server']['port']), specification_dir='.') - api = app.add_api('local.swagger.yml', debug=True, strict_validation=True, arguments={'host': f'{settings["server"]["host"]}:{settings["server"]["port"]}'}) +@click.command() +@click.pass_context +@click.option('--host', '-h', default='localhost', help='Hostname') +@click.option('--port', '-p', default=5000, help='port') +@click.option('--debug', '-d', default=False, is_flag=True, help='debug') +def serve(ctx, host, port, debug=False): + """Serve pygeoapi via Flask""" + + if port is not None: + port_ = port + else: + port_ = settings['server']['port'] + if host is not None: + host_ = host + else: + host_ = settings['server']['host'] + + app = connexion.FlaskApp(__name__, port=port_, specification_dir='.') + + hostport = '{}:{}'.format(host_, port_) + + api = app.add_api(settings['swagger'], debug=debug, strict_validation=True, + arguments={'host': hostport}) + settings['api'] = api.specification + + setup_logger() app.run() diff --git a/pygeoapi/log.py b/pygeoapi/log.py new file mode 100644 index 0000000..6736373 --- /dev/null +++ b/pygeoapi/log.py @@ -0,0 +1,72 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2018 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging +import sys + +from pygeoapi.config import settings + +LOGGER = logging.getLogger(__name__) + + +def setup_logger(): + """ + Setup configuration + + :returns: void (creates logging instance) + """ + + log_format = \ + '[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s' + + loglevels = { + 'CRITICAL': logging.CRITICAL, + 'ERROR': logging.ERROR, + 'WARNING': logging.WARNING, + 'INFO': logging.INFO, + 'DEBUG': logging.DEBUG, + 'NOTSET': logging.NOTSET, + } + formatter = logging.Formatter(log_format) + + log_handler = logging.NullHandler() + + if 'level' in settings['logging']: + loglevel = loglevels[settings['logging']['level']] + log_handler = logging.StreamHandler(sys.stdout) + + if 'logfile' in settings['logging']: + log_handler = logging.FileHandler(settings['logging']['logfile']) + + log_handler.setLevel(loglevel) + log_handler.setFormatter(formatter) + + LOGGER.addHandler(log_handler) + LOGGER.debug('Logging initialized') + return diff --git a/pygeoapi/provider/__init__.py b/pygeoapi/provider/__init__.py index ef2b1a5..228e72f 100644 --- a/pygeoapi/provider/__init__.py +++ b/pygeoapi/provider/__init__.py @@ -28,6 +28,9 @@ # ================================================================= import importlib +import logging + +LOGGER = logging.getLogger(__name__) PROVIDERS = { 'CSV': 'pygeoapi.provider.csv.CSVProvider', @@ -45,11 +48,16 @@ def load_provider(provider_obj): :returns: provider object """ + LOGGER.debug('Providers: {}'.format(PROVIDERS)) provider_name = provider_obj['type'] if provider_name not in PROVIDERS.keys(): - raise InvalidProviderError('provider not found') + msg = 'Provider {} not found'.format(provider_name) + LOGGER.exception(msg) + raise InvalidProviderError(msg) packagename, classname = PROVIDERS[provider_name].rsplit('.', 1) + LOGGER.debug('package name: {}'.format(packagename)) + LOGGER.debug('class name: {}'.format(classname)) module = importlib.import_module(packagename) class_ = getattr(module, classname) diff --git a/pygeoapi/provider/base.py b/pygeoapi/provider/base.py index 4d8cc65..b8c7e7d 100644 --- a/pygeoapi/provider/base.py +++ b/pygeoapi/provider/base.py @@ -27,23 +27,35 @@ # # ================================================================= +import logging from urllib.request import urlparse +LOGGER = logging.getLogger(__name__) + class BaseProvider(object): """generic Provider ABC""" def __init__(self, definition): - """initializer""" + """ + Initialize object + + :param definition: dict of dataset data definition + + :returns: pygeoapi.providers.base.BaseProvider + """ self.type = definition['type'] self.id_field = definition['id_field'] parsed_url = urlparse(definition['url']) + LOGGER.debug('Parsed URL: {}'.format(parsed_url)) if parsed_url.scheme == 'file': + LOGGER.debug('URL is a file') self.url = parsed_url.path else: + LOGGER.debug('URL is really a URL') self.url = definition['url'] def query(self): diff --git a/pygeoapi/provider/csv.py b/pygeoapi/provider/csv.py index cbabac6..b98822f 100644 --- a/pygeoapi/provider/csv.py +++ b/pygeoapi/provider/csv.py @@ -29,65 +29,88 @@ import csv import itertools +import logging from shapely import wkt from shapely.geometry import mapping from pygeoapi.provider.base import BaseProvider +LOGGER = logging.getLogger(__name__) + class CSVProvider(BaseProvider): """CSV provider""" def __init__(self, definition): - """initializer""" + """ + Initialize object + + :param definition: dict of dataset data definition + + :returns: pygeoapi.providers.csv.CSVProvider + """ BaseProvider.__init__(self, definition) - with open(self.url) as ff: - self.data = csv.DictReader(ff) + def _load(self, startindex=0, count=10, resulttype='results', + identifier=None): + """ + Load CSV data + + :param startindex: starting record to return (default 0) + :param count: number of records to return (default 10) + :param resulttype: return results or hit count (default results) + + :returns: dict of GeoJSON FeatureCollection + """ - def _load(self, identifier=None): feature_collection = { 'type': 'FeatureCollection', 'features': [] } with open(self.url) as ff: + LOGGER.debug('Serializing DictReader') data = csv.DictReader(ff) - feat = None - for row in data: + if resulttype == 'hits': + LOGGER('Retrurning hits only') + feature_collection['numberMatched'] = len(list(data)) + return feature_collection + LOGGER.debug('Slicing CSV rows') + for row in itertools.islice(data, startindex, startindex+count): feature = {'type': 'Feature'} feature['ID'] = row.pop('id') feature['geometry'] = mapping(wkt.loads(row.pop('geom'))) feature['properties'] = row if identifier is not None and feature['ID'] == identifier: - feat = feature - break - + return feature feature_collection['features'].append(feature) - if identifier is not None: - return feat - else: - return feature_collection + + return feature_collection def query(self, startindex=0, count=10, resulttype='results'): """ CSV query - :returns: dict of 0..n GeoJSON features + :param startindex: starting record to return (default 0) + :param count: number of records to return (default 10) + :param resulttype: return results or hit count (default results) + + :returns: dict of GeoJSON FeatureCollection """ - return self._load() + return self._load(startindex, count, resulttype) def get(self, identifier): """ query CSV id :param identifier: feature id + :returns: dict of single GeoJSON feature """ - return self._load(identifier) + return self._load(identifier=identifier) def __repr__(self): return ' {}'.format(self.url) diff --git a/pygeoapi/provider/elasticsearch.py b/pygeoapi/provider/elasticsearch.py index 17b560a..1a2cf38 100644 --- a/pygeoapi/provider/elasticsearch.py +++ b/pygeoapi/provider/elasticsearch.py @@ -27,30 +27,49 @@ # # ================================================================= +import logging + from elasticsearch import Elasticsearch from pygeoapi.provider.base import BaseProvider +LOGGER = logging.getLogger(__name__) + class ElasticsearchProvider(BaseProvider): """Elasticsearch Provider""" def __init__(self, definition): - """initializer""" + """ + Initialize object + + :param definition: dict of dataset data definition + + :returns: pygeoapi.providers.elasticsearch.ElasticsearchProvider + """ BaseProvider.__init__(self, definition) url_tokens = self.url.split('/') + LOGGER.debug('Setting Elasticsearch properties') + self.es_host = url_tokens[2] self.index_name = url_tokens[-2] self.type_name = url_tokens[-1] - self.es_host = url_tokens[2] + LOGGER.debug('host: {}'.format(self.es_host)) + LOGGER.debug('index: {}'.format(self.index_name)) + LOGGER.debug('type: {}'.format(self.type_name)) + LOGGER.debug('Connecting to Elasticsearch') self.es = Elasticsearch(self.es_host) def query(self, startindex=0, count=10, resulttype='results'): """ - query ES + query Elasticsearch index + + :param startindex: starting record to return (default 0) + :param count: number of records to return (default 10) + :param resulttype: return results or hit count (default results) :returns: dict of 0..n GeoJSON features """ @@ -60,14 +79,20 @@ class ElasticsearchProvider(BaseProvider): 'features': [] } + LOGGER.debug('Querying Elasticsearch') + if resulttype == 'hits': + LOGGER.debug('hits only specified') + count = 0 results = self.es.search(index=self.index_name, from_=startindex, size=count) if resulttype == 'hits': feature_collection['numberMatched'] = results['hits']['total'] return feature_collection + LOGGER.debug('serializing features') for feature in results['hits']['hits']: id_ = feature['_source']['properties']['identifier'] + LOGGER.debug('serializing id {}'.format(id_)) feature['_source']['ID'] = id_ feature_collection['features'].append(feature['_source']) @@ -78,15 +103,19 @@ class ElasticsearchProvider(BaseProvider): Get ES document by id :param identifier: feature id + :returns: dict of single GeoJSON feature """ try: + LOGGER.debug('Fetching identifier {}'.format(identifier)) result = self.es.get(self.index_name, doc_type=self.type_name, id=identifier) + LOGGER.debug('Serializing feature') id_ = result['_source']['properties']['identifier'] result['_source']['ID'] = id_ except Exception as err: + LOGGER.error(err) return None return result['_source'] diff --git a/pygeoapi/views.py b/pygeoapi/views.py index 0ba0858..1238c9d 100644 --- a/pygeoapi/views.py +++ b/pygeoapi/views.py @@ -28,57 +28,148 @@ # # ================================================================= +import logging + +from flask import request +# from flask import request, url_for from pygeoapi.config import settings from pygeoapi.provider import load_provider +LOGGER = logging.getLogger(__name__) + + def describe_collections(f='json'): + """ + Provide feature collection metadata + + :param f: response format (default JSON) + + :returns: dict of feature collection metadata + """ + # TODO allow other file return formats - if f.upper() == 'JSON': - fcm = { - 'collections': [] - } + if f.upper() != 'JSON': + msg = 'Unsupported format: {}'.format(f) + LOGGER.error(msg) + return msg, 400 - for k, v in settings['datasets'].items(): - collection = {'links': [], 'crs': []} - collection['collectionId'] = k - collection['title'] = v['title'] - collection['description'] = v['abstract'] - for crs in v['crs']: - collection['crs'].append( - 'http://www.opengis.net/def/crs/OGC/1.3/{}'.format(crs)) - collection['extent'] = v['extents']['spatial']['bbox'] + fcm = { + 'links': [], + 'collections': [] + } - for link in v['links']: - lnk = {'rel': link['type'], 'href': link['url']} - collection['links'].append(lnk) + url = '{}://{}'.format(request.scheme, settings['server']['host']) + if settings['server']['port'] not in [80, 443]: + url = '{}:{}'.format(url, settings['server']['port']) - fcm['collections'].append(collection) +# LOGGER.debug('Creating links') +# fcm['links'] = [{ +# 'rel': 'self', +# 'type': 'application/json', +# 'title': 'this document', +# 'href': '{}{}'.format(url, url_for('index_json')), +# }, { +# 'rel': 'self', +# 'type': 'text/html', +# 'title': 'this document as HTML', +# 'href': '{}{}'.format(url, url_for('index_html')), +# }, { +# 'rel': 'self', +# 'type': 'application/openapi+json;version=3.0', +# 'title': 'the OpenAPI definition as JSON', +# 'href': '{}{}'.format(url, url_for('api_json')), +# }, { +# 'rel': 'self', +# 'type': 'text/html', +# 'title': 'the OpenAPI definition as HTML', +# 'href': '{}{}'.format(url, url_for('api_html')), +# } +# ] + + LOGGER.debug('Creating collections') + for k, v in settings['datasets'].items(): + collection = {'links': [], 'crs': []} + collection['name'] = k + collection['title'] = v['title'] + collection['description'] = v['abstract'] + for crs in v['crs']: + collection['crs'].append( + 'http://www.opengis.net/def/crs/OGC/1.3/{}'.format(crs)) + collection['extent'] = v['extents']['spatial']['bbox'] + + for link in v['links']: + lnk = {'rel': link['type'], 'href': link['url']} + collection['links'].append(lnk) + + fcm['collections'].append(collection) + + return fcm - return {'collections' : collection} - else: - return f'"{f}" not supported as a query parameter.', 400 def get_specification(f='json'): if f.upper() == 'JSON': return settings['api'] else: - return f'"{f}" not supported as a query parameter.', 400 + return '{} not supported as a query parameter'.format(f), 400 -def getFeatures(ds, start_index, count, result_type, bbox=None, f='json'): - if ds not in settings['datasets'].keys(): - return f'dataset {ds} not found.', 400 - else: - p = load_provider(settings['datasets'][ds]['data']) - return p.query() -def getFeature(ds, id, f='json'): - if ds not in settings['datasets'].keys(): - return f'dataset {ds} not found.', 400 +def get_features(dataset, startindex=0, count=10, resulttype='results', + bbox=None, f='json'): + """ + Queries feature collection + + :param dataset: dataset to query + :param startindex: starting record to return (default 0) + :param count: number of records to return (default 10) + :param resulttype: return results or hit count (default results) + :param bbox: list of minx,miny,maxx,maxy + :param f: responase format (default GeoJSON) + + :returns: dict of GeoJSON FeatureCollection + + """ + if dataset not in settings['datasets'].keys(): + msg = 'dataset {} not found'.format(dataset) + LOGGER.error(msg) + return msg, 400 else: - p = load_provider(settings['datasets'][ds]['data']) - feat = p.get(id) - if feat is None: - return f'feature "{id}" not found', 404 - else: - return feat + LOGGER.debug('Loading provider') + p = load_provider(settings['datasets'][dataset]['data']) + LOGGER.debug('Querying provider') + LOGGER.debug('startindex: {}'.format(startindex)) + LOGGER.debug('count: {}'.format(count)) + LOGGER.debug('resulttype: {}'.format(resulttype)) + results = p.query(startindex=int(startindex), count=int(count), + resulttype=resulttype) + + return results + + +def get_feature(dataset, id, f='json'): + """ + Get a single feature + + :param dataset: dataset to query + :param id: feature identifier + :param f: responase format (default GeoJSON) + + :returns: dict of GeoJSON Feature + + """ + if dataset not in settings['datasets'].keys(): + msg = 'dataset {} not found'.format(dataset) + LOGGER.error(msg) + return msg, 400 + + LOGGER.debug('Loading provider') + p = load_provider(settings['datasets'][dataset]['data']) + + LOGGER.debug('Fetching id {}'.format(id)) + feature = p.get(id) + if feature is None: + msg = 'feature {} not found'.format(id) + LOGGER.warning(msg) + return msg, 404 + + return feature diff --git a/requirements.txt b/requirements.txt index f6ab5dc..5e6e648 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -connexion click +connexion elasticsearch flask flask_cors +pyyaml requests shapely -pyyaml diff --git a/tests/test_geojson_provider.py b/tests/test_geojson_provider.py index 42b66cc..9093f58 100644 --- a/tests/test_geojson_provider.py +++ b/tests/test_geojson_provider.py @@ -114,7 +114,6 @@ def test_update_safe_id(fixture, config): assert 'Null' in results['features'][0]['properties']['name'] - """ def __init__(self, definition): BaseProvider.__init__(self, definition)