implement queryables endpoint (#417)

* implement queryables endpoint

* fix tests now that CSV provider provides properties

* filter queryables if provider properties are set

docs/source/tour.rst

@@ -52,6 +52,16 @@ with related links (other related HTML pages, dataset download, etc.).
 
 The 'View' section provides the default to start browsing the data.
 
+The 'Queryables' section provides a link to the dataset's properties.
+
+
+Collection queryables
+---------------------
+
+http://localhost:5000/collections/obs/queryables
+
+The queryables endpoint provides a list of queryable properties and their associated datatypes.
+
 
 Collection items
 ----------------

pygeoapi/api.py

@@ -448,6 +448,86 @@ class API(object):
 
         return headers_, 200, json.dumps(fcm, default=json_serial)
 
+    @pre_process
+    @jsonldify
+    def get_collection_queryables(self, headers_, format_, dataset=None):
+        """
+        Provide collection queryables
+
+        :param headers_: copy of HEADERS object
+        :param format_: format of requests,
+                        pre checked by pre_process decorator
+        :param dataset: name of collection
+
+        :returns: tuple of headers, status code, content
+        """
+
+        if format_ is not None and format_ not in FORMATS:
+            exception = {
+                'code': 'InvalidParameterValue',
+                'description': 'Invalid format'
+            }
+            LOGGER.error(exception)
+            return headers_, 400, json.dumps(exception)
+
+        if any([dataset is None,
+                dataset not in self.config['datasets'].keys()]):
+
+            exception = {
+                'code': 'InvalidParameterValue',
+                'description': 'Invalid collection'
+            }
+            LOGGER.error(exception)
+            return headers_, 400, json.dumps(exception)
+
+        LOGGER.debug('Creating collection queryables')
+        LOGGER.debug('Loading provider')
+        try:
+            p = load_plugin('provider',
+                            self.config['datasets'][dataset]['provider'])
+        except ProviderConnectionError:
+            exception = {
+                'code': 'NoApplicableCode',
+                'description': 'connection error (check logs)'
+            }
+            LOGGER.error(exception)
+            return headers_, 500, json.dumps(exception)
+        except ProviderQueryError:
+            exception = {
+                'code': 'NoApplicableCode',
+                'description': 'query error (check logs)'
+            }
+            LOGGER.error(exception)
+            return headers_, 500, json.dumps(exception)
+
+        queryables = {
+            'queryables': []
+        }
+
+        for k, v in p.fields.items():
+            show_field = False
+            if p.properties:
+                if k in p.properties:
+                    show_field = True
+            else:
+                show_field = True
+
+            if show_field:
+                queryables['queryables'].append({
+                    'queryable': k,
+                    'type': v['type']
+                })
+
+        if format_ == 'html':  # render
+            queryables['title'] = self.config['datasets'][dataset]['title']
+            headers_['Content-Type'] = 'text/html'
+            content = render_j2_template(self.config, 'queryables.html',
+                                         queryables)
+
+            return headers_, 200, content
+
+        return headers_, 200, json.dumps(queryables, default=json_serial)
+
     def get_collection_items(self, headers, args, dataset, pathinfo=None):
         """
         Queries feature collection

pygeoapi/flask_app.py

@@ -99,6 +99,7 @@ def root():
     headers, status_code, content = api_.root(request.headers, request.args)
 
     response = make_response(content, status_code)
+
     if headers:
         response.headers = headers
 
@@ -119,6 +120,7 @@ def openapi():
                                                  openapi)
 
     response = make_response(content, status_code)
+
     if headers:
         response.headers = headers
 
@@ -137,6 +139,7 @@ def conformance():
                                                      request.args)
 
     response = make_response(content, status_code)
+
     if headers:
         response.headers = headers
 
@@ -147,9 +150,10 @@ def conformance():
 @APP.route('/collections/<name>')
 def describe_collections(name=None):
     """
-    OGC open api collections  access point
+    OGC open api collections access point
 
     :param name: identifier of collection name
+
     :returns: HTTP response
     """
 
@@ -157,6 +161,28 @@ def describe_collections(name=None):
         request.headers, request.args, name)
 
     response = make_response(content, status_code)
+
+    if headers:
+        response.headers = headers
+
+    return response
+
+
+@APP.route('/collections/<name>/queryables')
+def get_collection_queryables(name=None):
+    """
+    OGC open api collections querybles access point
+
+    :param name: identifier of collection name
+
+    :returns: HTTP response
+    """
+
+    headers, status_code, content = api_.get_collection_queryables(
+        request.headers, request.args, name)
+
+    response = make_response(content, status_code)
+
     if headers:
         response.headers = headers
 

pygeoapi/openapi.py

@@ -338,6 +338,26 @@ def get_oas_30(cfg):
 
         p = load_plugin('provider', cfg['datasets'][k]['provider'])
 
+        if p.fields:
+            queryables_path = '{}/queryables'.format(collection_name_path)
+
+            paths[queryables_path] = {
+                'get': {
+                    'summary': 'Get {} queryables'.format(v['title']),
+                    'description': v['description'],
+                    'tags': [k],
+                    'parameters': [
+                        items_f,
+                    ],
+                    'responses': {
+                        200: {'$ref': '{}#/components/responses/Features'.format(OPENAPI_YAML['oapif'])},  # noqa
+                        400: {'$ref': '{}#/components/responses/InvalidParameter'.format(OPENAPI_YAML['oapif'])},  # noqa
+                        404: {'$ref': '{}#/components/responses/NotFound'.format(OPENAPI_YAML['oapif'])},  # noqa
+                        500: {'$ref': '{}#/components/responses/ServerError'.format(OPENAPI_YAML['oapif'])}  # noqa
+                    }
+                }
+            }
+
         if p.time_field is not None:
             paths[items_path]['get']['parameters'].append(
                 {'$ref': '{}#/components/parameters/datetime'.format(OPENAPI_YAML['oapif'])})  # noqa

pygeoapi/provider/csv_.py

@@ -53,6 +53,25 @@ class CSVProvider(BaseProvider):
         BaseProvider.__init__(self, provider_def)
         self.geometry_x = provider_def['geometry']['x_field']
         self.geometry_y = provider_def['geometry']['y_field']
+        self.fields = self.get_fields()
+
+    def get_fields(self):
+        """
+         Get provider field information (names, types)
+
+        :returns: dict of fields
+        """
+
+        LOGGER.debug('Treating all columns as string types')
+        with open(self.data) as ff:
+            LOGGER.debug('Serializing DictReader')
+            data_ = csv.DictReader(ff)
+            fields = {}
+            for f in data_.fieldnames:
+                fields[f] = {
+                    'type': 'string'
+                }
+            return fields
 
     def _load(self, startindex=0, limit=10, resulttype='results',
               identifier=None, bbox=[], datetime=None, properties=[]):

pygeoapi/provider/geojson.py

@@ -65,6 +65,25 @@ class GeoJSONProvider(BaseProvider):
     def __init__(self, provider_def):
         """initializer"""
         BaseProvider.__init__(self, provider_def)
+        self.fields = self.get_fields()
+
+    def get_fields(self):
+        """
+         Get provider field information (names, types)
+
+        :returns: dict of fields
+        """
+
+        LOGGER.debug('Treating all columns as string types')
+        if os.path.exists(self.data):
+            with open(self.data) as src:
+                data = json.loads(src.read())
+            fields = {}
+            for f in data['features'][0]['properties'].keys():
+                fields[f] = {
+                    'type': 'string'
+                }
+            return fields
 
     def _load(self):
         """Load and validate the source GeoJSON file

pygeoapi/starlette_app.py

@@ -151,6 +151,29 @@ async def describe_collections(request: Request, name=None):
     return response
 
 
+@app.route('/collections/{name}/queryables')
+@app.route('/collections/{name}/queryables/')
+async def get_collection_queryables(request: Request, name=None):
+    """
+    OGC open api collections queryables access point
+
+    :param name: identifier of collection name
+
+    :returns: Starlette HTTP Response
+    """
+
+    if 'name' in request.path_params:
+        name = request.path_params['name']
+    headers, status_code, content = api_.get_collection_queryables(
+        request.headers, request.query_params, name)
+
+    response = Response(content=content, status_code=status_code)
+    if headers:
+        response.headers.update(headers)
+
+    return response
+
+
 @app.route('/collections/{feature_collection}/items')
 @app.route('/collections/{feature_collection}/items/')
 @app.route('/collections/{feature_collection}/items/{feature}')

pygeoapi/templates/collection.html

@@ -19,6 +19,15 @@
           <mark class="tag">{{ kw }}</mark>
         {% endfor %}
       </p>
+      <h3>Queryables</h3>
+      <ul>
+        <li>
+          <div>
+            <meta itemprop="encodingFormat" content="text/html" />
+            <a title="Display Queryables" itemprop="contentURL" href="{{ config['server']['url'] }}/collections/{{ data['id'] }}/queryables">
+              Display Queryables of "{{ data['title'] }}</a>"</div>
+        </li>
+      </ul>
       <h3>View</h3>
       <ul>
         <li>

pygeoapi/templates/queryables.html

@@ -0,0 +1,30 @@
+{% extends "base.html" %}
+{% block title %}{{ super() }} {{ data['title'] }} {% endblock %}
+{% block crumbs %}{{ super() }} 
+/ <a href="../../collections">Collections</a>
+/ <a href="./{{ data['id'] }}">{{ data['title'] }}</a>
+/ <a href="./{{ data['id'] }}queryables">Queryables</a>
+{% endblock %}
+{% block body %}
+    <section id="collection" itemscope itemtype="https://schema.org/Dataset">
+      <span itemprop="includedInDataCatalog" itemscope itemtype="https://schema.org/DataCatalog">
+        <meta itemprop="url" content="{{ config['server']['url'] }}/collections" />
+        <meta itemprop="name" content="{{ config['metadata']['identification']['title'] | striptags }}" />
+        <meta itemprop="description" content="{{ config['metadata']['identification']['description'] | striptags }}" />
+      </span>
+      <h1 itemprop="name">{{ data['title'] }}</h1>
+      <meta itemprop="url" content="{{ config['server']['url'] }}" />
+      <p itemprop="description">{{ data['description'] }}</p>
+      <p itemprop="keywords">
+        {% for kw in data['keywords'] %}
+          <mark class="tag">{{ kw }}</mark>
+        {% endfor %}
+      </p>
+      <h3>Queryables</h3>
+      <ul>
+      {% for queryable in data['queryables'] %}
+        <li>{{ queryable['queryable'] }} (<code>{{ queryable['type'] }}</code>)</li>
+      {% endfor %}
+      </ul>
+    </section>
+{% endblock %}

tests/test_api.py

@@ -227,6 +227,35 @@ def test_describe_collections(config, api_):
     assert rsp_headers['Content-Type'] == 'text/html'
 
 
+def test_get_collection_queryables(config, api_):
+    req_headers = make_req_headers()
+    rsp_headers, code, response = api_.get_collection_queryables(
+        req_headers, {}, 'notfound')
+    assert code == 400
+
+    req_headers = make_req_headers()
+    rsp_headers, code, response = api_.get_collection_queryables(
+        req_headers, {'f': 'html'}, 'obs')
+    assert rsp_headers['Content-Type'] == 'text/html'
+
+    rsp_headers, code, response = api_.get_collection_queryables(
+        req_headers, {'f': 'json'}, 'obs')
+    queryables = json.loads(response)
+
+    assert 'queryables' in queryables
+    assert len(queryables['queryables']) == 6
+
+    # test with provider filtered properties
+    api_.config['datasets']['obs']['provider']['properties'] = ['stn_id']
+
+    rsp_headers, code, response = api_.get_collection_queryables(
+        req_headers, {'f': 'json'}, 'obs')
+    queryables = json.loads(response)
+
+    assert 'queryables' in queryables
+    assert len(queryables['queryables']) == 1
+
+
 def test_describe_collections_json_ld(config, api_):
     req_headers = make_req_headers()
     rsp_headers, code, response = api_.describe_collections(
@@ -387,7 +416,7 @@ def test_get_collection_items(config, api_):
 
     rsp_headers, code, response = api_.get_collection_items(
         req_headers, {
-            'sortby': 'stn_id',
+            'sortby': 'bad-property',
             'stn_id': '35'
         }, 'obs')
 
@@ -406,7 +435,7 @@ def test_get_collection_items(config, api_):
         req_headers, {'sortby': 'stn_id:A'}, 'obs')
     features = json.loads(response)
     # FIXME? this test errors out currently
-    assert code == 400
+    assert code == 200
 
     rsp_headers, code, response = api_.get_collection_items(
         req_headers, {'f': 'csv'}, 'obs')

tests/test_csv__provider.py

@@ -65,6 +65,12 @@ def config():
 
 def test_query(fixture, config):
     p = CSVProvider(config)
+
+    fields = p.get_fields()
+    assert len(fields) == 6
+    assert fields['value']['type'] == 'string'
+    assert fields['stn_id']['type'] == 'string'
+
     results = p.query()
     assert len(results['features']) == 5
     assert results['numberMatched'] == 5

tests/test_elasticsearch__provider.py

@@ -44,6 +44,13 @@ def config():
 
 def test_query(config):
     p = ElasticsearchProvider(config)
+
+    fields = p.get_fields()
+    assert len(fields) == 37
+    assert fields['scalerank']['type'] == 'long'
+    assert fields['changed']['type'] == 'float'
+    assert fields['ls_name']['type'] == 'string'
+
     results = p.query()
     assert len(results['features']) == 10
     assert results['numberMatched'] == 242

tests/test_geojson_provider.py

@@ -67,6 +67,12 @@ def config():
 
 def test_query(fixture, config):
     p = GeoJSONProvider(config)
+
+    fields = p.get_fields()
+    assert len(fields) == 2
+    assert fields['id']['type'] == 'string'
+    assert fields['name']['type'] == 'string'
+
     results = p.query()
     assert len(results['features']) == 1
     assert results['numberMatched'] == 1