diff --git a/.travis.yml b/.travis.yml index a16df73..38e009f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,10 +17,11 @@ install: env: - PYGEOAPI_CONFIG=pygeoapi-config.yml -before_script: - - pygeoapi generate_openapi_document -c pygeoapi-config.yml > pygeoapi-openapi.yml +#before_script: +# - pygeoapi generate_openapi_document -c pygeoapi-config.yml > pygeoapi-openapi.yml script: + - pygeoapi generate_openapi_document -c pygeoapi-config.yml > pygeoapi-openapi.yml - pytest --cov=pygeoapi - find . -type f -name "*.py" | xargs flake8 - python setup.py --long-description | rst2html5.py diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml index 1f6a840..7319750 100644 --- a/pygeoapi-config.yml +++ b/pygeoapi-config.yml @@ -11,7 +11,7 @@ server: limit: 10 logging: - level: INFO + level: ERROR #logfile: /tmp/pygeoapi.log metadata: diff --git a/pygeoapi/api.py b/pygeoapi/api.py index a05775e..7dd06da 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -274,8 +274,19 @@ class API(object): 'Content-type': 'application/json' } + properties = [] + reserved_fieldnames = ['bbox', 'f', 'limit', 'startindex', + 'resulttype', 'time'] formats = ['json', 'html'] + if dataset not in self.config['datasets'].keys(): + exception = { + 'code': 'InvalidParameterValue', + 'description': 'Invalid feature collection' + } + LOGGER.error(exception) + return headers_, 400, json.dumps(exception) + format_ = args.get('f') if format_ is not None and format_ not in formats: exception = { @@ -311,16 +322,14 @@ class API(object): time = args.get('time') - if dataset not in self.config['datasets'].keys(): - exception = { - 'code': 'InvalidParameterValue', - 'description': 'Invalid feature collection' - } - LOGGER.error(exception) - return headers_, 400, json.dumps(exception) - LOGGER.debug('Loading provider') p = load_provider(self.config['datasets'][dataset]['provider']) + + LOGGER.debug('processing property parameters') + for k, v in args.items(): + if k in reserved_fieldnames: + properties.append((k, v)) + LOGGER.debug('Querying provider') LOGGER.debug('startindex: {}'.format(startindex)) LOGGER.debug('limit: {}'.format(limit)) @@ -328,7 +337,8 @@ class API(object): try: content = p.query(startindex=int(startindex), limit=int(limit), - resulttype=resulttype, bbox=bbox, time=time) + resulttype=resulttype, bbox=bbox, time=time, + properties=properties) except ProviderConnectionError: exception = { 'code': 'NoApplicableCode', diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index f4dda61..311ac7b 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -32,6 +32,8 @@ import logging import click import yaml +from pygeoapi.provider import load_provider + LOGGER = logging.getLogger(__name__) @@ -116,7 +118,7 @@ def get_oas_30(cfg): paths['/collections'] = { 'get': { 'summary': 'Feature Collections', - 'descriptions': 'Feature Collections', + 'description': 'Feature Collections', 'tags': ['server'], 'responses': { 200: { @@ -136,6 +138,7 @@ def get_oas_30(cfg): ) LOGGER.debug('setting up datasets') for k, v in cfg['datasets'].items(): + collection_name_path = '/collections/{}'.format(k) tag = { 'name': k, 'description': v['description'], @@ -146,10 +149,12 @@ def get_oas_30(cfg): tag['externalDocs']['description'] = link['type'] tag['externalDocs']['url'] = link['url'] break + if len(tag['externalDocs']) == 0: + del tag['externalDocs'] oas['tags'].append(tag) - paths['/collections/{}'.format(k)] = { + paths[collection_name_path] = { 'get': { 'summary': 'Get feature collection metadata'.format(v['title']), # noqa 'description': v['description'], @@ -168,7 +173,7 @@ def get_oas_30(cfg): } } - paths['/collections/{}/items'.format(k)] = { + paths['{}/items'.format(collection_name_path)] = { 'get': { 'summary': 'Get {} features'.format(v['title']), 'description': v['description'], @@ -190,7 +195,22 @@ def get_oas_30(cfg): } } - paths['/collections/{}/items/{{id}}'.format(k)] = { + p = load_provider(cfg['datasets'][k]['provider']) + + for k2, v2 in p.fields.items(): + path_ = '{}/items'.format(collection_name_path) + paths['{}/items'.format(path_)]['get']['parameters'].append({ + 'name': k2, + 'in': 'query', + 'required': False, + 'schema': { + 'type': v2['type'], + }, + 'style': 'form', + 'explode': False + }) + + paths['{}/items/{{id}}'.format(collection_name_path)] = { 'get': { 'summary': 'Get {} feature by ID'.format(v['title']), 'description': v['description'], @@ -220,21 +240,14 @@ def get_oas_30(cfg): 'in': 'path', 'description': 'The id of a feature', 'required': True, - 'type': 'string' + 'schema': { + 'type': 'string' + } }, 'limit': { 'name': 'limit', 'in': 'query', - 'description': ('The optional limit parameter limits the', - ' number of items that are presented in the', - ' response document. Only items are counted', - ' that are on the first level of the', - ' collection in the response document. Nested', - ' objects contained within the explicitly', - ' requested items shall not be counted.', - ' Minimum = 1. Maximum = 10000.', - ' Default = {}.'.format( - cfg['server']['limit'])), + 'description': 'The optional limit parameter limits the number of items that are presented in the response document. Only items are counted that are on the first level of the collection in the response document. Nested objects contained within the explicitly requested items shall not be counted. Minimum = 1. Maximum = 10000. Default = {}.'.format(cfg['server']['limit']), # noqa 'required': False, 'schema': { 'type': 'integer', @@ -277,4 +290,4 @@ def generate_openapi_document(ctx, config_file): raise click.ClickException('--config/-c required') with open(config_file) as ff: s = yaml.load(ff) - click.echo(yaml.dump(get_oas(s), default_flow_style=False)) + click.echo(yaml.safe_dump(get_oas(s), default_flow_style=False)) diff --git a/pygeoapi/provider/base.py b/pygeoapi/provider/base.py index 8d64a21..0138ea5 100644 --- a/pygeoapi/provider/base.py +++ b/pygeoapi/provider/base.py @@ -48,6 +48,16 @@ class BaseProvider(object): self.data = provider_def['data'] self.id_field = provider_def['id_field'] self.time_field = provider_def.get('time_field') + self.fields = {} + + def get_fields(self): + """ + Get provider field information (names, types) + + :returns: dict of fields + """ + + raise NotImplementedError() def query(self): """ diff --git a/pygeoapi/provider/csv_.py b/pygeoapi/provider/csv_.py index ba3bdba..046f8a3 100644 --- a/pygeoapi/provider/csv_.py +++ b/pygeoapi/provider/csv_.py @@ -54,13 +54,14 @@ class CSVProvider(BaseProvider): BaseProvider.__init__(self, provider_def) def _load(self, startindex=0, limit=10, resulttype='results', - identifier=None, bbox=[], time=None): + identifier=None, bbox=[], time=None, properties=[]): """ Load CSV data :param startindex: starting record to return (default 0) :param limit: number of records to return (default 10) :param resulttype: return results or hit limit (default results) + :param properties: list of tuples (name, value) :returns: dict of GeoJSON FeatureCollection """ @@ -101,13 +102,16 @@ class CSVProvider(BaseProvider): return feature_collection def query(self, startindex=0, limit=10, resulttype='results', - bbox=[], time=None): + bbox=[], time=None, properties=[]): """ CSV query :param startindex: starting record to return (default 0) :param limit: number of records to return (default 10) :param resulttype: return results or hit limit (default results) + :param bbox: bounding box [minx,miny,maxx,maxy] + :param time: temporal (datestamp or extent) + :param properties: list of tuples (name, value) :returns: dict of GeoJSON FeatureCollection """ diff --git a/pygeoapi/provider/elasticsearch_.py b/pygeoapi/provider/elasticsearch_.py index bb10084..a42482a 100644 --- a/pygeoapi/provider/elasticsearch_.py +++ b/pygeoapi/provider/elasticsearch_.py @@ -30,6 +30,7 @@ import logging from elasticsearch import Elasticsearch, exceptions +from elasticsearch.client.indices import IndicesClient from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError, ProviderQueryError) @@ -63,9 +64,32 @@ class ElasticsearchProvider(BaseProvider): LOGGER.debug('Connecting to Elasticsearch') self.es = Elasticsearch(self.es_host) + LOGGER.debug('Grabbing field information') + self.fields = self.get_fields() + + def get_fields(self): + """ + Get provider field information (names, types) + + :returns: dict of fields + """ + + fields_ = {} + ic = IndicesClient(self.es) + ii = ic.get(self.index_name) + p = ii[self.index_name]['mappings'][self.type_name]['properties']['properties'] # noqa + + for k, v in p['properties'].items(): + if v['type'] == 'text': + type_ = 'string' + else: + type_ = v['type'] + fields_[k] = {'type': type_} + + return fields_ def query(self, startindex=0, limit=10, resulttype='results', - bbox=[], time=None): + bbox=[], time=None, properties=[]): """ query Elasticsearch index @@ -74,6 +98,7 @@ class ElasticsearchProvider(BaseProvider): :param resulttype: return results or hit limit (default results) :param bbox: bounding box [minx,miny,maxx,maxy] :param time: temporal (datestamp or extent) + :param properties: list of tuples (name, value) :returns: dict of 0..n GeoJSON features """ @@ -137,8 +162,18 @@ class ElasticsearchProvider(BaseProvider): LOGGER.debug(filter_) query['query']['bool']['filter'].append(filter_) + if properties: + LOGGER.debug('processing properties') + for prop in properties: + pf = { + 'match': { + 'properties.{}'.format(prop[0]): prop[1] + } + } + query['query']['bool']['filter'].append(pf) + try: - LOGGER.debug('Querying Elasticsearch') + LOGGER.debug('querying Elasticsearch') results = self.es.search(index=self.index_name, from_=startindex, size=limit, body=query) except exceptions.ConnectionError as err: diff --git a/pygeoapi/provider/geojson.py b/pygeoapi/provider/geojson.py index c316913..9402a75 100644 --- a/pygeoapi/provider/geojson.py +++ b/pygeoapi/provider/geojson.py @@ -90,11 +90,17 @@ class GeoJSONProvider(BaseProvider): return data def query(self, startindex=0, limit=10, resulttype='results', - bbox=[], time=None): + bbox=[], time=None, properties=[]): """ query the provider - :param bbox: Bounding Box in [W, S, E, N] order + :param startindex: starting record to return (default 0) + :param limit: number of records to return (default 10) + :param resulttype: return results or hit limit (default results) + :param bbox: bounding box [minx,miny,maxx,maxy] + :param time: temporal (datestamp or extent) + :param properties: list of tuples (name, value) + :returns: FeatureCollection dict of 0..n GeoJSON features """ # TODO filter by bbox without resorting to third-party libs diff --git a/pygeoapi/provider/sqlite.py b/pygeoapi/provider/sqlite.py index bec45a7..f49d7e3 100644 --- a/pygeoapi/provider/sqlite.py +++ b/pygeoapi/provider/sqlite.py @@ -143,7 +143,7 @@ class SQLiteProvider(BaseProvider): return cursor def query(self, startindex=0, limit=10, resulttype='results', - bbox=[], time=None): + bbox=[], time=None, properties=[]): """ Query Sqlite for all the content. e,g: http://localhost:5000/collections/countries/items? @@ -152,6 +152,9 @@ class SQLiteProvider(BaseProvider): :param startindex: starting record to return (default 0) :param limit: number of records to return (default 10) :param resulttype: return results or hit limit (default results) + :param bbox: bounding box [minx,miny,maxx,maxy] + :param time: temporal (datestamp or extent) + :param properties: list of tuples (name, value) :returns: GeoJSON FeaturesCollection """ diff --git a/pygeoapi/templates/items.html b/pygeoapi/templates/items.html index b957084..b2cb78a 100644 --- a/pygeoapi/templates/items.html +++ b/pygeoapi/templates/items.html @@ -18,13 +18,13 @@

{{ data['title'] }}

{{ data['description'] }} -

Features JSON

+

Features JSON

@@ -48,7 +48,7 @@ var geojson_data = {{ data['features'] |to_json }}; var items = new L.GeoJSON(geojson_data, { onEachFeature: function (feature, layer) { - var html_ = '' + feature.ID + ''; + var html_ = '' + feature.ID + ''; layer.bindPopup(html_); } });