add support for propertyname and skipGeometry parameters (#51) (#565)

* add support for propertyname and skipGeometry parameters (#51)

* fix tests

* fix tests

* fix tests

* fix vars

* add tests

docs/source/data-publishing/ogcapi-coverages.rst

@@ -18,8 +18,8 @@ parameters.
    :header: Provider, rangeSubset, subset, bbox, datetime
    :align: left
 
-   rasterio,✔️,✔️,✔️,
-   xarray,✔️,✔️,✔️,✔️
+   rasterio,✅,✅,✅,
+   xarray,✅,✅,✅,✅
 
 
 Below are specific connection examples based on supported providers.

docs/source/data-publishing/ogcapi-features.rst

@@ -15,16 +15,16 @@ pygeoapi core feature providers are listed below, along with a matrix of support
 parameters.
 
 .. csv-table::
-   :header: Provider, properties, resulttype, bbox, datetime, sortby
+   :header: Provider, properties (filters), resulttype, bbox, datetime, sortby, properties (display)
    :align: left
 
-   CSV,✔️ ,results/hits,❌,❌,❌
-   Elasticsearch,✔️ ,results/hits,✔️ ,✔️ ,✔️ 
-   GeoJSON,✔️ ,results/hits,❌,❌,❌
-   MongoDB,✔️ ,results,✔️ ,✔️ ,✔️ 
-   OGR,✔️ ,results/hits,✔️ ,❌,❌
-   PostgreSQL,✔️ ,results/hits,✔️ ,❌,❌
-   SQLiteGPKG,✔️ ,results/hits,✔️ ,❌,❌
+   CSV,✅,results/hits,❌,❌,❌,✅
+   Elasticsearch,✅,results/hits,✅,✅,✅,✅
+   GeoJSON,✅,results/hits,❌,❌,❌,❌
+   MongoDB,✅,results,✅,✅,✅,❌
+   OGR,✅,results/hits,✅,❌,❌,❌
+   PostgreSQL,✅,results/hits,✅,❌,❌,❌
+   SQLiteGPKG,✅,results/hits,✅,❌,❌,❌
 
 
 Below are specific connection examples based on supported providers.

docs/source/data-publishing/ogcapi-tiles.rst

@@ -23,7 +23,7 @@ pygeoapi core tile providers are listed below, along with supported storage type
    :header: Provider, local, remote
    :align: left
 
-   MVT,✔️,✔️
+   MVT,✅,✅
 
 
 Below are specific connection examples based on supported providers.

pygeoapi/api.py

@@ -57,8 +57,8 @@ from pygeoapi.provider.tile import (ProviderTileNotFoundError,
                                     ProviderTilesetIdNotFoundError)
 from pygeoapi.util import (dategetter, filter_dict_by_key_value,
                            get_provider_by_type, get_provider_default,
-                           get_typed_value, render_j2_template, TEMPLATES,
-                           to_json)
+                           get_typed_value, render_j2_template, str2bool,
+                           TEMPLATES, to_json)
 
 LOGGER = logging.getLogger(__name__)
 
@@ -705,7 +705,8 @@ class API:
 
         properties = []
         reserved_fieldnames = ['bbox', 'f', 'limit', 'startindex',
-                               'resulttype', 'datetime', 'sortby']
+                               'resulttype', 'datetime', 'sortby',
+                               'properties', 'skipGeometry']
         formats = FORMATS
         formats.extend(f.lower() for f in PLUGINS['formatter'].keys())
 
@@ -881,6 +882,31 @@ class API:
         else:
             sortby = []
 
+        LOGGER.debug('processing properties parameter')
+        val = args.get('properties')
+
+        if val is not None:
+            select_properties = val.split(',')
+            properties_to_check = set(p.properties) | set(p.fields.keys())
+
+            if (len(list(set(select_properties) -
+                         set(properties_to_check))) > 0):
+                exception = {
+                    'code': 'InvalidParameterValue',
+                    'description': 'unknown properties specified'
+                }
+                LOGGER.error(exception)
+                return headers_, 400, to_json(exception, self.pretty_print)
+        else:
+            select_properties = []
+
+        LOGGER.debug('processing skipGeometry parameter')
+        val = args.get('skipGeometry')
+        if val is not None:
+            skip_geometry = str2bool(val)
+        else:
+            skip_geometry = False
+
         LOGGER.debug('Querying provider')
         LOGGER.debug('startindex: {}'.format(startindex))
         LOGGER.debug('limit: {}'.format(limit))
@@ -888,12 +914,16 @@ class API:
         LOGGER.debug('sortby: {}'.format(sortby))
         LOGGER.debug('bbox: {}'.format(bbox))
         LOGGER.debug('datetime: {}'.format(datetime_))
+        LOGGER.debug('properties: {}'.format(select_properties))
+        LOGGER.debug('skipGeometry: {}'.format(skip_geometry))
 
         try:
             content = p.query(startindex=startindex, limit=limit,
                               resulttype=resulttype, bbox=bbox,
                               datetime_=datetime_, properties=properties,
-                              sortby=sortby)
+                              sortby=sortby,
+                              select_properties=select_properties,
+                              skip_geometry=skip_geometry)
         except ProviderConnectionError as err:
             exception = {
                 'code': 'NoApplicableCode',

pygeoapi/openapi.py

@@ -269,6 +269,32 @@ def get_oas_30(cfg):
                 'style': 'form',
                 'explode': False
             },
+            'properties': {
+                'name': 'properties',
+                'in': 'query',
+                'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.',  # noqa
+                'required': False,
+                'style': 'form',
+                'explode': False,
+                'schema': {
+                    'type': 'array',
+                    'items': {
+                        'type': 'string'
+                    }
+                }
+            },
+            'skipGeometry': {
+                'name': 'skipGeometry',
+                'in': 'query',
+                'description': 'This option can be used to skip response geometries for each feature.',  # noqa
+                'required': False,
+                'style': 'form',
+                'explode': False,
+                'schema': {
+                    'type': 'boolean',
+                    'default': False
+                }
+            },
             'sortby': {
                 'name': 'sortby',
                 'in': 'query',
@@ -397,6 +423,10 @@ def get_oas_30(cfg):
 
             items_path = '{}/items'.format(collection_name_path)
 
+            coll_properties = deepcopy(oas['components']['parameters']['properties'])  # noqa
+
+            coll_properties['schema']['items']['enum'] = list(p.fields.keys())
+
             paths[items_path] = {
                 'get': {
                     'summary': 'Get {} items'.format(v['title']),
@@ -407,6 +437,8 @@ def get_oas_30(cfg):
                         items_f,
                         {'$ref': '{}#/components/parameters/bbox'.format(OPENAPI_YAML['oapif'])},  # noqa
                         {'$ref': '{}#/components/parameters/limit'.format(OPENAPI_YAML['oapif'])},  # noqa
+                        coll_properties,
+                        {'$ref': '#/components/parameters/skipGeometry'},
                         {'$ref': '#/components/parameters/sortby'},
                         {'$ref': '#/components/parameters/startindex'}
                     ],

pygeoapi/provider/csv_.py

@@ -72,7 +72,8 @@ class CSVProvider(BaseProvider):
             return fields
 
     def _load(self, startindex=0, limit=10, resulttype='results',
-              identifier=None, bbox=[], datetime_=None, properties=[]):
+              identifier=None, bbox=[], datetime_=None, properties=[],
+              select_properties=[], skip_geometry=False):
         """
         Load CSV data
 
@@ -81,6 +82,8 @@ class CSVProvider(BaseProvider):
         :param datetime_: temporal (datestamp or extent)
         :param resulttype: return results or hit limit (default results)
         :param properties: list of tuples (name, value)
+        :param select_properties: list of property names
+        :param skip_geometry: bool of whether to skip geometry (default False)
 
         :returns: dict of GeoJSON FeatureCollection
         """
@@ -103,16 +106,19 @@ class CSVProvider(BaseProvider):
             for row in itertools.islice(data_, startindex, startindex+limit):
                 feature = {'type': 'Feature'}
                 feature['id'] = row.pop(self.id_field)
-                feature['geometry'] = {
-                    'type': 'Point',
-                    'coordinates': [
-                        float(row.pop(self.geometry_x)),
-                        float(row.pop(self.geometry_y))
-                    ]
-                }
-                if self.properties:
+                if not skip_geometry:
+                    feature['geometry'] = {
+                        'type': 'Point',
+                        'coordinates': [
+                            float(row.pop(self.geometry_x)),
+                            float(row.pop(self.geometry_y))
+                        ]
+                    }
+                else:
+                    feature['geometry'] = None
+                if self.properties or select_properties:
                     feature['properties'] = OrderedDict()
-                    for p in self.properties:
+                    for p in set(self.properties) | set(select_properties):
                         try:
                             feature['properties'][p] = row[p]
                         except KeyError as err:
@@ -139,7 +145,8 @@ class CSVProvider(BaseProvider):
         return feature_collection
 
     def query(self, startindex=0, limit=10, resulttype='results',
-              bbox=[], datetime_=None, properties=[], sortby=[]):
+              bbox=[], datetime_=None, properties=[], sortby=[],
+              select_properties=[], skip_geometry=False):
         """
         CSV query
 
@@ -150,11 +157,15 @@ class CSVProvider(BaseProvider):
         :param datetime_: temporal (datestamp or extent)
         :param properties: list of tuples (name, value)
         :param sortby: list of dicts (property, order)
+        :param select_properties: list of property names
+        :param skip_geometry: bool of whether to skip geometry (default False)
 
         :returns: dict of GeoJSON FeatureCollection
         """
 
-        return self._load(startindex, limit, resulttype)
+        return self._load(startindex, limit, resulttype,
+                          select_properties=select_properties,
+                          skip_geometry=skip_geometry)
 
     def get(self, identifier):
         """

pygeoapi/provider/elasticsearch_.py

@@ -115,7 +115,8 @@ class ElasticsearchProvider(BaseProvider):
         return fields_
 
     def query(self, startindex=0, limit=10, resulttype='results',
-              bbox=[], datetime_=None, properties=[], sortby=[]):
+              bbox=[], datetime_=None, properties=[], sortby=[],
+              select_properties=[], skip_geometry=False):
         """
         query Elasticsearch index
 
@@ -126,6 +127,8 @@ class ElasticsearchProvider(BaseProvider):
         :param datetime_: temporal (datestamp or extent)
         :param properties: list of tuples (name, value)
         :param sortby: list of dicts (property, order)
+        :param select_properties: list of property names
+        :param skip_geometry: bool of whether to skip geometry (default False)
 
         :returns: dict of 0..n GeoJSON features
         """
@@ -228,15 +231,23 @@ class ElasticsearchProvider(BaseProvider):
                 }
                 query['sort'].append(sort_)
 
-        if self.properties:
+        if self.properties or select_properties:
             LOGGER.debug('including specified fields: {}'.format(
                 self.properties))
             query['_source'] = {
-                'includes': list(map(self.mask_prop, self.properties))
+                'includes': list(map(self.mask_prop,
+                                 set(self.properties) | set(select_properties)))  # noqa
             }
             query['_source']['includes'].append(self.mask_prop(self.id_field))
             query['_source']['includes'].append('type')
             query['_source']['includes'].append('geometry')
+        if skip_geometry:
+            LOGGER.debug('limiting to specified fields: {}'.format(
+                select_properties))
+            try:
+                query['_source']['excludes'] = ['geometry']
+            except KeyError:
+                query['_source'] = {'excludes': ['geometry']}
         try:
             LOGGER.debug('querying Elasticsearch')
             LOGGER.debug(json.dumps(query, indent=4))
@@ -352,7 +363,7 @@ class ElasticsearchProvider(BaseProvider):
             if 'type' not in doc['_source']:
                 feature_['id'] = id_
                 feature_['type'] = 'Feature'
-            feature_['geometry'] = doc['_source']['geometry']
+            feature_['geometry'] = doc['_source'].get('geometry')
             feature_['properties'] = {}
             for key, value in doc['_source'].items():
                 if key == 'geometry':
@@ -363,12 +374,13 @@ class ElasticsearchProvider(BaseProvider):
             feature_ = doc['_source']
             id_ = doc['_source']['properties'][self.id_field]
             feature_['id'] = id_
+            feature_['geometry'] = doc['_source'].get('geometry')
 
         if self.properties:
             feature_thinned = {
                 'id': id_,
                 'type': feature_['type'],
-                'geometry': feature_['geometry'],
+                'geometry': feature_.get('geometry'),
                 'properties': OrderedDict()
             }
             for p in self.properties:

pygeoapi/provider/geojson.py

@@ -84,7 +84,7 @@ class GeoJSONProvider(BaseProvider):
                 fields[f] = 'string'
             return fields
 
-    def _load(self):
+    def _load(self, skip_geometry=None, select_properties=[]):
         """Load and validate the source GeoJSON file
         at self.data
 
@@ -106,10 +106,16 @@ class GeoJSONProvider(BaseProvider):
         for i in data['features']:
             if 'id' not in i and self.id_field in i['properties']:
                 i['id'] = i['properties'][self.id_field]
+            if skip_geometry:
+                i['geometry'] = None
+            if self.properties or select_properties:
+                i['properties'] = {k: v for k, v in i['properties'].items()
+                                   if k in set(self.properties) | set(select_properties)}  # noqa
         return data
 
     def query(self, startindex=0, limit=10, resulttype='results',
-              bbox=[], datetime_=None, properties=[], sortby=[]):
+              bbox=[], datetime_=None, properties=[], sortby=[],
+              select_properties=[], skip_geometry=False):
         """
         query the provider
 
@@ -120,12 +126,15 @@ class GeoJSONProvider(BaseProvider):
         :param datetime_: temporal (datestamp or extent)
         :param properties: list of tuples (name, value)
         :param sortby: list of dicts (property, order)
+        :param select_properties: list of property names
+        :param skip_geometry: bool of whether to skip geometry (default False)
 
         :returns: FeatureCollection dict of 0..n GeoJSON features
         """
 
         # TODO filter by bbox without resorting to third-party libs
-        data = self._load()
+        data = self._load(skip_geometry=skip_geometry,
+                          select_properties=select_properties)
 
         data['numberMatched'] = len(data['features'])
 

tests/test_api.py

@@ -503,6 +503,16 @@ def test_get_collection_items(config, api_):
 
     assert code == 200
 
+    rsp_headers, code, response = api_.get_collection_items(
+        req_headers, {'skipGeometry': 'true'}, 'obs')
+
+    assert json.loads(response)['features'][0]['geometry'] is None
+
+    rsp_headers, code, response = api_.get_collection_items(
+        req_headers, {'properties': 'foo,bar'}, 'obs')
+
+    assert code == 400
+
 
 def test_get_collection_items_json_ld(config, api_):
     req_headers = make_req_headers()
@@ -915,6 +925,6 @@ def test_validate_datetime():
             '2001-10-30/2002-10-30')
 
     with pytest.raises(ValueError):
-        _ = validate_datetime(config, '2000/..')
+        _ = validate_datetime(config, '1999/..')
     with pytest.raises(ValueError):
         _ = validate_datetime(config, '../2010')

tests/test_csv__provider.py

@@ -89,6 +89,15 @@ def test_query(config):
 
     assert len(results['features'][0]['properties']) == 3
 
+    results = p.query(select_properties=['value'])
+    assert len(results['features'][0]['properties']) == 1
+
+    results = p.query(select_properties=['value', 'stn_id'])
+    assert len(results['features'][0]['properties']) == 2
+
+    results = p.query(skip_geometry=True)
+    assert results['features'][0]['geometry'] is None
+
     config['properties'] = ['value', 'stn_id']
     p = CSVProvider(config)
     results = p.query()

tests/test_elasticsearch__provider.py

@@ -93,6 +93,15 @@ def test_query(config):
     assert results['numberMatched'] == 242
     assert results['numberReturned'] == 242
 
+    results = p.query(select_properties=['nameascii'])
+    assert len(results['features'][0]['properties']) == 2
+
+    results = p.query(select_properties=['nameascii', 'scalerank'])
+    assert len(results['features'][0]['properties']) == 3
+
+    results = p.query(skip_geometry=True)
+    assert results['features'][0]['geometry'] is None
+
     config['properties'] = ['nameascii']
     p = ElasticsearchProvider(config)
     results = p.query()

tests/test_geojson_provider.py

@@ -49,7 +49,11 @@ def fixture():
                 'type': 'Point',
                 'coordinates': [125.6, 10.1]},
             'properties': {
-                'name': 'Dinagat Islands'}}]}
+                'name': 'Dinagat Islands',
+                'foo': 'bar'
+            }}
+        ]
+    }
 
     with open(path, 'w') as fh:
         fh.write(json.dumps(data))
@@ -70,7 +74,7 @@ def test_query(fixture, config):
     p = GeoJSONProvider(config)
 
     fields = p.get_fields()
-    assert len(fields) == 1
+    assert len(fields) == 2
     assert fields['name'] == 'string'
 
     results = p.query()
@@ -79,6 +83,12 @@ def test_query(fixture, config):
     assert results['numberReturned'] == 1
     assert results['features'][0]['id'] == '123-456'
 
+    results = p.query(select_properties=['foo'])
+    assert len(results['features'][0]['properties']) == 1
+
+    results = p.query(skip_geometry=True)
+    assert results['features'][0]['geometry'] is None
+
 
 def test_get(fixture, config):
     p = GeoJSONProvider(config)