From af5aa82e26724d3cb1adb5a9ed2e9c3662cdbba3 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sat, 21 Apr 2018 00:19:11 +0000 Subject: [PATCH 1/6] add properties support --- pygeoapi-config.yml | 2 +- pygeoapi/api.py | 35 ++++++++++++++++------ pygeoapi/openapi.py | 45 +++++++++++++++++++---------- pygeoapi/provider/base.py | 10 +++++++ pygeoapi/provider/csv_.py | 8 +++-- pygeoapi/provider/elasticsearch_.py | 39 +++++++++++++++++++++++-- pygeoapi/provider/geojson.py | 10 +++++-- pygeoapi/provider/sqlite.py | 5 +++- pygeoapi/templates/items.html | 6 ++-- 9 files changed, 124 insertions(+), 36 deletions(-) diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml index 1f6a840..7319750 100644 --- a/pygeoapi-config.yml +++ b/pygeoapi-config.yml @@ -11,7 +11,7 @@ server: limit: 10 logging: - level: INFO + level: ERROR #logfile: /tmp/pygeoapi.log metadata: diff --git a/pygeoapi/api.py b/pygeoapi/api.py index a05775e..58c9ce3 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -274,8 +274,19 @@ class API(object): 'Content-type': 'application/json' } + properties = [] + reserved_fieldnames = ['bbox', 'f', 'limit', 'startindex', + 'resulttype', 'time'] formats = ['json', 'html'] + if dataset not in self.config['datasets'].keys(): + exception = { + 'code': 'InvalidParameterValue', + 'description': 'Invalid feature collection' + } + LOGGER.error(exception) + return headers_, 400, json.dumps(exception) + format_ = args.get('f') if format_ is not None and format_ not in formats: exception = { @@ -311,16 +322,21 @@ class API(object): time = args.get('time') - if dataset not in self.config['datasets'].keys(): - exception = { - 'code': 'InvalidParameterValue', - 'description': 'Invalid feature collection' - } - LOGGER.error(exception) - return headers_, 400, json.dumps(exception) - LOGGER.debug('Loading provider') p = load_provider(self.config['datasets'][dataset]['provider']) + + LOGGER.debug('processing property parameters') + for k, v in args.items(): + if k not in reserved_fieldnames: + if k not in p.fields.keys(): + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid property' + } + LOGGER.error('invalid property: {}'.format(k)) + return headers_, 400, json.dumps(exception) + properties.append((k, v)) + LOGGER.debug('Querying provider') LOGGER.debug('startindex: {}'.format(startindex)) LOGGER.debug('limit: {}'.format(limit)) @@ -328,7 +344,8 @@ class API(object): try: content = p.query(startindex=int(startindex), limit=int(limit), - resulttype=resulttype, bbox=bbox, time=time) + resulttype=resulttype, bbox=bbox, time=time, + properties=properties) except ProviderConnectionError: exception = { 'code': 'NoApplicableCode', diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index f4dda61..311ac7b 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -32,6 +32,8 @@ import logging import click import yaml +from pygeoapi.provider import load_provider + LOGGER = logging.getLogger(__name__) @@ -116,7 +118,7 @@ def get_oas_30(cfg): paths['/collections'] = { 'get': { 'summary': 'Feature Collections', - 'descriptions': 'Feature Collections', + 'description': 'Feature Collections', 'tags': ['server'], 'responses': { 200: { @@ -136,6 +138,7 @@ def get_oas_30(cfg): ) LOGGER.debug('setting up datasets') for k, v in cfg['datasets'].items(): + collection_name_path = '/collections/{}'.format(k) tag = { 'name': k, 'description': v['description'], @@ -146,10 +149,12 @@ def get_oas_30(cfg): tag['externalDocs']['description'] = link['type'] tag['externalDocs']['url'] = link['url'] break + if len(tag['externalDocs']) == 0: + del tag['externalDocs'] oas['tags'].append(tag) - paths['/collections/{}'.format(k)] = { + paths[collection_name_path] = { 'get': { 'summary': 'Get feature collection metadata'.format(v['title']), # noqa 'description': v['description'], @@ -168,7 +173,7 @@ def get_oas_30(cfg): } } - paths['/collections/{}/items'.format(k)] = { + paths['{}/items'.format(collection_name_path)] = { 'get': { 'summary': 'Get {} features'.format(v['title']), 'description': v['description'], @@ -190,7 +195,22 @@ def get_oas_30(cfg): } } - paths['/collections/{}/items/{{id}}'.format(k)] = { + p = load_provider(cfg['datasets'][k]['provider']) + + for k2, v2 in p.fields.items(): + path_ = '{}/items'.format(collection_name_path) + paths['{}/items'.format(path_)]['get']['parameters'].append({ + 'name': k2, + 'in': 'query', + 'required': False, + 'schema': { + 'type': v2['type'], + }, + 'style': 'form', + 'explode': False + }) + + paths['{}/items/{{id}}'.format(collection_name_path)] = { 'get': { 'summary': 'Get {} feature by ID'.format(v['title']), 'description': v['description'], @@ -220,21 +240,14 @@ def get_oas_30(cfg): 'in': 'path', 'description': 'The id of a feature', 'required': True, - 'type': 'string' + 'schema': { + 'type': 'string' + } }, 'limit': { 'name': 'limit', 'in': 'query', - 'description': ('The optional limit parameter limits the', - ' number of items that are presented in the', - ' response document. Only items are counted', - ' that are on the first level of the', - ' collection in the response document. Nested', - ' objects contained within the explicitly', - ' requested items shall not be counted.', - ' Minimum = 1. Maximum = 10000.', - ' Default = {}.'.format( - cfg['server']['limit'])), + 'description': 'The optional limit parameter limits the number of items that are presented in the response document. Only items are counted that are on the first level of the collection in the response document. Nested objects contained within the explicitly requested items shall not be counted. Minimum = 1. Maximum = 10000. Default = {}.'.format(cfg['server']['limit']), # noqa 'required': False, 'schema': { 'type': 'integer', @@ -277,4 +290,4 @@ def generate_openapi_document(ctx, config_file): raise click.ClickException('--config/-c required') with open(config_file) as ff: s = yaml.load(ff) - click.echo(yaml.dump(get_oas(s), default_flow_style=False)) + click.echo(yaml.safe_dump(get_oas(s), default_flow_style=False)) diff --git a/pygeoapi/provider/base.py b/pygeoapi/provider/base.py index 8d64a21..0138ea5 100644 --- a/pygeoapi/provider/base.py +++ b/pygeoapi/provider/base.py @@ -48,6 +48,16 @@ class BaseProvider(object): self.data = provider_def['data'] self.id_field = provider_def['id_field'] self.time_field = provider_def.get('time_field') + self.fields = {} + + def get_fields(self): + """ + Get provider field information (names, types) + + :returns: dict of fields + """ + + raise NotImplementedError() def query(self): """ diff --git a/pygeoapi/provider/csv_.py b/pygeoapi/provider/csv_.py index ba3bdba..046f8a3 100644 --- a/pygeoapi/provider/csv_.py +++ b/pygeoapi/provider/csv_.py @@ -54,13 +54,14 @@ class CSVProvider(BaseProvider): BaseProvider.__init__(self, provider_def) def _load(self, startindex=0, limit=10, resulttype='results', - identifier=None, bbox=[], time=None): + identifier=None, bbox=[], time=None, properties=[]): """ Load CSV data :param startindex: starting record to return (default 0) :param limit: number of records to return (default 10) :param resulttype: return results or hit limit (default results) + :param properties: list of tuples (name, value) :returns: dict of GeoJSON FeatureCollection """ @@ -101,13 +102,16 @@ class CSVProvider(BaseProvider): return feature_collection def query(self, startindex=0, limit=10, resulttype='results', - bbox=[], time=None): + bbox=[], time=None, properties=[]): """ CSV query :param startindex: starting record to return (default 0) :param limit: number of records to return (default 10) :param resulttype: return results or hit limit (default results) + :param bbox: bounding box [minx,miny,maxx,maxy] + :param time: temporal (datestamp or extent) + :param properties: list of tuples (name, value) :returns: dict of GeoJSON FeatureCollection """ diff --git a/pygeoapi/provider/elasticsearch_.py b/pygeoapi/provider/elasticsearch_.py index bb10084..a42482a 100644 --- a/pygeoapi/provider/elasticsearch_.py +++ b/pygeoapi/provider/elasticsearch_.py @@ -30,6 +30,7 @@ import logging from elasticsearch import Elasticsearch, exceptions +from elasticsearch.client.indices import IndicesClient from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError, ProviderQueryError) @@ -63,9 +64,32 @@ class ElasticsearchProvider(BaseProvider): LOGGER.debug('Connecting to Elasticsearch') self.es = Elasticsearch(self.es_host) + LOGGER.debug('Grabbing field information') + self.fields = self.get_fields() + + def get_fields(self): + """ + Get provider field information (names, types) + + :returns: dict of fields + """ + + fields_ = {} + ic = IndicesClient(self.es) + ii = ic.get(self.index_name) + p = ii[self.index_name]['mappings'][self.type_name]['properties']['properties'] # noqa + + for k, v in p['properties'].items(): + if v['type'] == 'text': + type_ = 'string' + else: + type_ = v['type'] + fields_[k] = {'type': type_} + + return fields_ def query(self, startindex=0, limit=10, resulttype='results', - bbox=[], time=None): + bbox=[], time=None, properties=[]): """ query Elasticsearch index @@ -74,6 +98,7 @@ class ElasticsearchProvider(BaseProvider): :param resulttype: return results or hit limit (default results) :param bbox: bounding box [minx,miny,maxx,maxy] :param time: temporal (datestamp or extent) + :param properties: list of tuples (name, value) :returns: dict of 0..n GeoJSON features """ @@ -137,8 +162,18 @@ class ElasticsearchProvider(BaseProvider): LOGGER.debug(filter_) query['query']['bool']['filter'].append(filter_) + if properties: + LOGGER.debug('processing properties') + for prop in properties: + pf = { + 'match': { + 'properties.{}'.format(prop[0]): prop[1] + } + } + query['query']['bool']['filter'].append(pf) + try: - LOGGER.debug('Querying Elasticsearch') + LOGGER.debug('querying Elasticsearch') results = self.es.search(index=self.index_name, from_=startindex, size=limit, body=query) except exceptions.ConnectionError as err: diff --git a/pygeoapi/provider/geojson.py b/pygeoapi/provider/geojson.py index c316913..9402a75 100644 --- a/pygeoapi/provider/geojson.py +++ b/pygeoapi/provider/geojson.py @@ -90,11 +90,17 @@ class GeoJSONProvider(BaseProvider): return data def query(self, startindex=0, limit=10, resulttype='results', - bbox=[], time=None): + bbox=[], time=None, properties=[]): """ query the provider - :param bbox: Bounding Box in [W, S, E, N] order + :param startindex: starting record to return (default 0) + :param limit: number of records to return (default 10) + :param resulttype: return results or hit limit (default results) + :param bbox: bounding box [minx,miny,maxx,maxy] + :param time: temporal (datestamp or extent) + :param properties: list of tuples (name, value) + :returns: FeatureCollection dict of 0..n GeoJSON features """ # TODO filter by bbox without resorting to third-party libs diff --git a/pygeoapi/provider/sqlite.py b/pygeoapi/provider/sqlite.py index bec45a7..f49d7e3 100644 --- a/pygeoapi/provider/sqlite.py +++ b/pygeoapi/provider/sqlite.py @@ -143,7 +143,7 @@ class SQLiteProvider(BaseProvider): return cursor def query(self, startindex=0, limit=10, resulttype='results', - bbox=[], time=None): + bbox=[], time=None, properties=[]): """ Query Sqlite for all the content. e,g: http://localhost:5000/collections/countries/items? @@ -152,6 +152,9 @@ class SQLiteProvider(BaseProvider): :param startindex: starting record to return (default 0) :param limit: number of records to return (default 10) :param resulttype: return results or hit limit (default results) + :param bbox: bounding box [minx,miny,maxx,maxy] + :param time: temporal (datestamp or extent) + :param properties: list of tuples (name, value) :returns: GeoJSON FeaturesCollection """ diff --git a/pygeoapi/templates/items.html b/pygeoapi/templates/items.html index b957084..b2cb78a 100644 --- a/pygeoapi/templates/items.html +++ b/pygeoapi/templates/items.html @@ -18,13 +18,13 @@

{{ data['title'] }}

{{ data['description'] }} -

Features JSON

+

Features JSON

@@ -48,7 +48,7 @@ var geojson_data = {{ data['features'] |to_json }}; var items = new L.GeoJSON(geojson_data, { onEachFeature: function (feature, layer) { - var html_ = '' + feature.ID + ''; + var html_ = '' + feature.ID + ''; layer.bindPopup(html_); } }); From c172799180587c581f17ef54059095d631d6f8a9 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 25 Apr 2018 01:49:01 +0000 Subject: [PATCH 2/6] start elasticsearch --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index a16df73..2d5114d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,9 @@ dist: xenial python: - "3.5" +services: + - elasticsearch + before_install: - sudo apt-get -qq update - sudo apt-get install -y libsqlite3-mod-spatialite pandoc devscripts From 97de3c9a244ea20fccc1ad74cd7784b30cb0e706 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 25 Apr 2018 01:50:44 +0000 Subject: [PATCH 3/6] test --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2d5114d..ba75dbf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,9 +5,6 @@ dist: xenial python: - "3.5" -services: - - elasticsearch - before_install: - sudo apt-get -qq update - sudo apt-get install -y libsqlite3-mod-spatialite pandoc devscripts @@ -20,8 +17,8 @@ install: env: - PYGEOAPI_CONFIG=pygeoapi-config.yml -before_script: - - pygeoapi generate_openapi_document -c pygeoapi-config.yml > pygeoapi-openapi.yml +#before_script: +# - pygeoapi generate_openapi_document -c pygeoapi-config.yml > pygeoapi-openapi.yml script: - pytest --cov=pygeoapi From 51b55c34464e83ac681df4a6d374463dbe517c8d Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 25 Apr 2018 02:08:39 +0000 Subject: [PATCH 4/6] fix --- .travis.yml | 1 + pygeoapi/api.py | 9 +-------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index ba75dbf..38e009f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ env: # - pygeoapi generate_openapi_document -c pygeoapi-config.yml > pygeoapi-openapi.yml script: + - pygeoapi generate_openapi_document -c pygeoapi-config.yml > pygeoapi-openapi.yml - pytest --cov=pygeoapi - find . -type f -name "*.py" | xargs flake8 - python setup.py --long-description | rst2html5.py diff --git a/pygeoapi/api.py b/pygeoapi/api.py index 58c9ce3..7dd06da 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -327,14 +327,7 @@ class API(object): LOGGER.debug('processing property parameters') for k, v in args.items(): - if k not in reserved_fieldnames: - if k not in p.fields.keys(): - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid property' - } - LOGGER.error('invalid property: {}'.format(k)) - return headers_, 400, json.dumps(exception) + if k in reserved_fieldnames: properties.append((k, v)) LOGGER.debug('Querying provider') From 4bb3ff94536f7048299eea1d88b4af8576a4c64f Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 25 Apr 2018 02:15:13 +0000 Subject: [PATCH 5/6] fix --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 38e009f..784fa3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python -dist: xenial +#dist: xenial python: - "3.5" From 911829e3489acbb8cfd3818db61a1211115a21cf Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 25 Apr 2018 02:18:10 +0000 Subject: [PATCH 6/6] fix --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 784fa3a..38e009f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python -#dist: xenial +dist: xenial python: - "3.5"