From 54732d26c9fb53a326b6649acf144643c666b778 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Thu, 23 Apr 2020 08:59:02 -0400 Subject: [PATCH] implement queryables endpoint (#417) * implement queryables endpoint * fix tests now that CSV provider provides properties * filter queryables if provider properties are set --- docs/source/tour.rst | 10 ++++ pygeoapi/api.py | 80 +++++++++++++++++++++++++++ pygeoapi/flask_app.py | 28 +++++++++- pygeoapi/openapi.py | 20 +++++++ pygeoapi/provider/csv_.py | 19 +++++++ pygeoapi/provider/geojson.py | 19 +++++++ pygeoapi/starlette_app.py | 23 ++++++++ pygeoapi/templates/collection.html | 9 +++ pygeoapi/templates/queryables.html | 30 ++++++++++ tests/test_api.py | 33 ++++++++++- tests/test_csv__provider.py | 6 ++ tests/test_elasticsearch__provider.py | 7 +++ tests/test_geojson_provider.py | 6 ++ 13 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 pygeoapi/templates/queryables.html diff --git a/docs/source/tour.rst b/docs/source/tour.rst index c2b14e5..4166db2 100644 --- a/docs/source/tour.rst +++ b/docs/source/tour.rst @@ -52,6 +52,16 @@ with related links (other related HTML pages, dataset download, etc.). The 'View' section provides the default to start browsing the data. +The 'Queryables' section provides a link to the dataset's properties. + + +Collection queryables +--------------------- + +http://localhost:5000/collections/obs/queryables + +The queryables endpoint provides a list of queryable properties and their associated datatypes. + Collection items ---------------- diff --git a/pygeoapi/api.py b/pygeoapi/api.py index 6b4028b..cb42ee3 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -448,6 +448,86 @@ class API(object): return headers_, 200, json.dumps(fcm, default=json_serial) + @pre_process + @jsonldify + def get_collection_queryables(self, headers_, format_, dataset=None): + """ + Provide collection queryables + + :param headers_: copy of HEADERS object + :param format_: format of requests, + pre checked by pre_process decorator + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + if format_ is not None and format_ not in FORMATS: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'Invalid format' + } + LOGGER.error(exception) + return headers_, 400, json.dumps(exception) + + if any([dataset is None, + dataset not in self.config['datasets'].keys()]): + + exception = { + 'code': 'InvalidParameterValue', + 'description': 'Invalid collection' + } + LOGGER.error(exception) + return headers_, 400, json.dumps(exception) + + LOGGER.debug('Creating collection queryables') + LOGGER.debug('Loading provider') + try: + p = load_plugin('provider', + self.config['datasets'][dataset]['provider']) + except ProviderConnectionError: + exception = { + 'code': 'NoApplicableCode', + 'description': 'connection error (check logs)' + } + LOGGER.error(exception) + return headers_, 500, json.dumps(exception) + except ProviderQueryError: + exception = { + 'code': 'NoApplicableCode', + 'description': 'query error (check logs)' + } + LOGGER.error(exception) + return headers_, 500, json.dumps(exception) + + queryables = { + 'queryables': [] + } + + for k, v in p.fields.items(): + show_field = False + if p.properties: + if k in p.properties: + show_field = True + else: + show_field = True + + if show_field: + queryables['queryables'].append({ + 'queryable': k, + 'type': v['type'] + }) + + if format_ == 'html': # render + queryables['title'] = self.config['datasets'][dataset]['title'] + headers_['Content-Type'] = 'text/html' + content = render_j2_template(self.config, 'queryables.html', + queryables) + + return headers_, 200, content + + return headers_, 200, json.dumps(queryables, default=json_serial) + def get_collection_items(self, headers, args, dataset, pathinfo=None): """ Queries feature collection diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index ded2fb9..a8e5277 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -99,6 +99,7 @@ def root(): headers, status_code, content = api_.root(request.headers, request.args) response = make_response(content, status_code) + if headers: response.headers = headers @@ -119,6 +120,7 @@ def openapi(): openapi) response = make_response(content, status_code) + if headers: response.headers = headers @@ -137,6 +139,7 @@ def conformance(): request.args) response = make_response(content, status_code) + if headers: response.headers = headers @@ -147,9 +150,10 @@ def conformance(): @APP.route('/collections/') def describe_collections(name=None): """ - OGC open api collections access point + OGC open api collections access point :param name: identifier of collection name + :returns: HTTP response """ @@ -157,6 +161,28 @@ def describe_collections(name=None): request.headers, request.args, name) response = make_response(content, status_code) + + if headers: + response.headers = headers + + return response + + +@APP.route('/collections//queryables') +def get_collection_queryables(name=None): + """ + OGC open api collections querybles access point + + :param name: identifier of collection name + + :returns: HTTP response + """ + + headers, status_code, content = api_.get_collection_queryables( + request.headers, request.args, name) + + response = make_response(content, status_code) + if headers: response.headers = headers diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index e213d97..e05efef 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -338,6 +338,26 @@ def get_oas_30(cfg): p = load_plugin('provider', cfg['datasets'][k]['provider']) + if p.fields: + queryables_path = '{}/queryables'.format(collection_name_path) + + paths[queryables_path] = { + 'get': { + 'summary': 'Get {} queryables'.format(v['title']), + 'description': v['description'], + 'tags': [k], + 'parameters': [ + items_f, + ], + 'responses': { + 200: {'$ref': '{}#/components/responses/Features'.format(OPENAPI_YAML['oapif'])}, # noqa + 400: {'$ref': '{}#/components/responses/InvalidParameter'.format(OPENAPI_YAML['oapif'])}, # noqa + 404: {'$ref': '{}#/components/responses/NotFound'.format(OPENAPI_YAML['oapif'])}, # noqa + 500: {'$ref': '{}#/components/responses/ServerError'.format(OPENAPI_YAML['oapif'])} # noqa + } + } + } + if p.time_field is not None: paths[items_path]['get']['parameters'].append( {'$ref': '{}#/components/parameters/datetime'.format(OPENAPI_YAML['oapif'])}) # noqa diff --git a/pygeoapi/provider/csv_.py b/pygeoapi/provider/csv_.py index 60f2848..206cebc 100644 --- a/pygeoapi/provider/csv_.py +++ b/pygeoapi/provider/csv_.py @@ -53,6 +53,25 @@ class CSVProvider(BaseProvider): BaseProvider.__init__(self, provider_def) self.geometry_x = provider_def['geometry']['x_field'] self.geometry_y = provider_def['geometry']['y_field'] + self.fields = self.get_fields() + + def get_fields(self): + """ + Get provider field information (names, types) + + :returns: dict of fields + """ + + LOGGER.debug('Treating all columns as string types') + with open(self.data) as ff: + LOGGER.debug('Serializing DictReader') + data_ = csv.DictReader(ff) + fields = {} + for f in data_.fieldnames: + fields[f] = { + 'type': 'string' + } + return fields def _load(self, startindex=0, limit=10, resulttype='results', identifier=None, bbox=[], datetime=None, properties=[]): diff --git a/pygeoapi/provider/geojson.py b/pygeoapi/provider/geojson.py index 05350de..3cc8f4e 100644 --- a/pygeoapi/provider/geojson.py +++ b/pygeoapi/provider/geojson.py @@ -65,6 +65,25 @@ class GeoJSONProvider(BaseProvider): def __init__(self, provider_def): """initializer""" BaseProvider.__init__(self, provider_def) + self.fields = self.get_fields() + + def get_fields(self): + """ + Get provider field information (names, types) + + :returns: dict of fields + """ + + LOGGER.debug('Treating all columns as string types') + if os.path.exists(self.data): + with open(self.data) as src: + data = json.loads(src.read()) + fields = {} + for f in data['features'][0]['properties'].keys(): + fields[f] = { + 'type': 'string' + } + return fields def _load(self): """Load and validate the source GeoJSON file diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 60e4c9e..47c73b6 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -151,6 +151,29 @@ async def describe_collections(request: Request, name=None): return response +@app.route('/collections/{name}/queryables') +@app.route('/collections/{name}/queryables/') +async def get_collection_queryables(request: Request, name=None): + """ + OGC open api collections queryables access point + + :param name: identifier of collection name + + :returns: Starlette HTTP Response + """ + + if 'name' in request.path_params: + name = request.path_params['name'] + headers, status_code, content = api_.get_collection_queryables( + request.headers, request.query_params, name) + + response = Response(content=content, status_code=status_code) + if headers: + response.headers.update(headers) + + return response + + @app.route('/collections/{feature_collection}/items') @app.route('/collections/{feature_collection}/items/') @app.route('/collections/{feature_collection}/items/{feature}') diff --git a/pygeoapi/templates/collection.html b/pygeoapi/templates/collection.html index ba105ef..485d788 100644 --- a/pygeoapi/templates/collection.html +++ b/pygeoapi/templates/collection.html @@ -19,6 +19,15 @@ {{ kw }} {% endfor %}

+

Queryables

+

View

  • diff --git a/pygeoapi/templates/queryables.html b/pygeoapi/templates/queryables.html new file mode 100644 index 0000000..b0ec7d7 --- /dev/null +++ b/pygeoapi/templates/queryables.html @@ -0,0 +1,30 @@ +{% extends "base.html" %} +{% block title %}{{ super() }} {{ data['title'] }} {% endblock %} +{% block crumbs %}{{ super() }} +/ Collections +/ {{ data['title'] }} +/ Queryables +{% endblock %} +{% block body %} +
    + + + + + +

    {{ data['title'] }}

    + +

    {{ data['description'] }}

    +

    + {% for kw in data['keywords'] %} + {{ kw }} + {% endfor %} +

    +

    Queryables

    +
      + {% for queryable in data['queryables'] %} +
    • {{ queryable['queryable'] }} ({{ queryable['type'] }})
      • + {% endfor %} +
    +
    +{% endblock %} diff --git a/tests/test_api.py b/tests/test_api.py index f860900..6373fa4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -227,6 +227,35 @@ def test_describe_collections(config, api_): assert rsp_headers['Content-Type'] == 'text/html' +def test_get_collection_queryables(config, api_): + req_headers = make_req_headers() + rsp_headers, code, response = api_.get_collection_queryables( + req_headers, {}, 'notfound') + assert code == 400 + + req_headers = make_req_headers() + rsp_headers, code, response = api_.get_collection_queryables( + req_headers, {'f': 'html'}, 'obs') + assert rsp_headers['Content-Type'] == 'text/html' + + rsp_headers, code, response = api_.get_collection_queryables( + req_headers, {'f': 'json'}, 'obs') + queryables = json.loads(response) + + assert 'queryables' in queryables + assert len(queryables['queryables']) == 6 + + # test with provider filtered properties + api_.config['datasets']['obs']['provider']['properties'] = ['stn_id'] + + rsp_headers, code, response = api_.get_collection_queryables( + req_headers, {'f': 'json'}, 'obs') + queryables = json.loads(response) + + assert 'queryables' in queryables + assert len(queryables['queryables']) == 1 + + def test_describe_collections_json_ld(config, api_): req_headers = make_req_headers() rsp_headers, code, response = api_.describe_collections( @@ -387,7 +416,7 @@ def test_get_collection_items(config, api_): rsp_headers, code, response = api_.get_collection_items( req_headers, { - 'sortby': 'stn_id', + 'sortby': 'bad-property', 'stn_id': '35' }, 'obs') @@ -406,7 +435,7 @@ def test_get_collection_items(config, api_): req_headers, {'sortby': 'stn_id:A'}, 'obs') features = json.loads(response) # FIXME? this test errors out currently - assert code == 400 + assert code == 200 rsp_headers, code, response = api_.get_collection_items( req_headers, {'f': 'csv'}, 'obs') diff --git a/tests/test_csv__provider.py b/tests/test_csv__provider.py index f3d7dd1..a2bcaa9 100644 --- a/tests/test_csv__provider.py +++ b/tests/test_csv__provider.py @@ -65,6 +65,12 @@ def config(): def test_query(fixture, config): p = CSVProvider(config) + + fields = p.get_fields() + assert len(fields) == 6 + assert fields['value']['type'] == 'string' + assert fields['stn_id']['type'] == 'string' + results = p.query() assert len(results['features']) == 5 assert results['numberMatched'] == 5 diff --git a/tests/test_elasticsearch__provider.py b/tests/test_elasticsearch__provider.py index 9ad5746..c5722dd 100644 --- a/tests/test_elasticsearch__provider.py +++ b/tests/test_elasticsearch__provider.py @@ -44,6 +44,13 @@ def config(): def test_query(config): p = ElasticsearchProvider(config) + + fields = p.get_fields() + assert len(fields) == 37 + assert fields['scalerank']['type'] == 'long' + assert fields['changed']['type'] == 'float' + assert fields['ls_name']['type'] == 'string' + results = p.query() assert len(results['features']) == 10 assert results['numberMatched'] == 242 diff --git a/tests/test_geojson_provider.py b/tests/test_geojson_provider.py index 0390bf4..aa77bf8 100644 --- a/tests/test_geojson_provider.py +++ b/tests/test_geojson_provider.py @@ -67,6 +67,12 @@ def config(): def test_query(fixture, config): p = GeoJSONProvider(config) + + fields = p.get_fields() + assert len(fields) == 2 + assert fields['id']['type'] == 'string' + assert fields['name']['type'] == 'string' + results = p.query() assert len(results['features']) == 1 assert results['numberMatched'] == 1