diff --git a/docs/source/data-publishing/ogcapi-coverages.rst b/docs/source/data-publishing/ogcapi-coverages.rst index e0bbb7e..bf29290 100644 --- a/docs/source/data-publishing/ogcapi-coverages.rst +++ b/docs/source/data-publishing/ogcapi-coverages.rst @@ -18,8 +18,8 @@ parameters. :header: Provider, rangeSubset, subset, bbox, datetime :align: left - rasterio,✔️,✔️,✔️, - xarray,✔️,✔️,✔️,✔️ + rasterio,✅,✅,✅, + xarray,✅,✅,✅,✅ Below are specific connection examples based on supported providers. diff --git a/docs/source/data-publishing/ogcapi-features.rst b/docs/source/data-publishing/ogcapi-features.rst index 45f7de6..4af081a 100644 --- a/docs/source/data-publishing/ogcapi-features.rst +++ b/docs/source/data-publishing/ogcapi-features.rst @@ -15,16 +15,16 @@ pygeoapi core feature providers are listed below, along with a matrix of support parameters. .. csv-table:: - :header: Provider, properties, resulttype, bbox, datetime, sortby + :header: Provider, properties (filters), resulttype, bbox, datetime, sortby, properties (display) :align: left - CSV,✔️ ,results/hits,❌,❌,❌ - Elasticsearch,✔️ ,results/hits,✔️ ,✔️ ,✔️ - GeoJSON,✔️ ,results/hits,❌,❌,❌ - MongoDB,✔️ ,results,✔️ ,✔️ ,✔️ - OGR,✔️ ,results/hits,✔️ ,❌,❌ - PostgreSQL,✔️ ,results/hits,✔️ ,❌,❌ - SQLiteGPKG,✔️ ,results/hits,✔️ ,❌,❌ + CSV,✅,results/hits,❌,❌,❌,✅ + Elasticsearch,✅,results/hits,✅,✅,✅,✅ + GeoJSON,✅,results/hits,❌,❌,❌,❌ + MongoDB,✅,results,✅,✅,✅,❌ + OGR,✅,results/hits,✅,❌,❌,❌ + PostgreSQL,✅,results/hits,✅,❌,❌,❌ + SQLiteGPKG,✅,results/hits,✅,❌,❌,❌ Below are specific connection examples based on supported providers. diff --git a/docs/source/data-publishing/ogcapi-tiles.rst b/docs/source/data-publishing/ogcapi-tiles.rst index 579967c..4b8d8f4 100644 --- a/docs/source/data-publishing/ogcapi-tiles.rst +++ b/docs/source/data-publishing/ogcapi-tiles.rst @@ -23,7 +23,7 @@ pygeoapi core tile providers are listed below, along with supported storage type :header: Provider, local, remote :align: left - MVT,✔️,✔️ + MVT,✅,✅ Below are specific connection examples based on supported providers. diff --git a/pygeoapi/api.py b/pygeoapi/api.py index b496d73..8d2d70c 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -57,8 +57,8 @@ from pygeoapi.provider.tile import (ProviderTileNotFoundError, ProviderTilesetIdNotFoundError) from pygeoapi.util import (dategetter, filter_dict_by_key_value, get_provider_by_type, get_provider_default, - get_typed_value, render_j2_template, TEMPLATES, - to_json) + get_typed_value, render_j2_template, str2bool, + TEMPLATES, to_json) LOGGER = logging.getLogger(__name__) @@ -705,7 +705,8 @@ class API: properties = [] reserved_fieldnames = ['bbox', 'f', 'limit', 'startindex', - 'resulttype', 'datetime', 'sortby'] + 'resulttype', 'datetime', 'sortby', + 'properties', 'skipGeometry'] formats = FORMATS formats.extend(f.lower() for f in PLUGINS['formatter'].keys()) @@ -881,6 +882,31 @@ class API: else: sortby = [] + LOGGER.debug('processing properties parameter') + val = args.get('properties') + + if val is not None: + select_properties = val.split(',') + properties_to_check = set(p.properties) | set(p.fields.keys()) + + if (len(list(set(select_properties) - + set(properties_to_check))) > 0): + exception = { + 'code': 'InvalidParameterValue', + 'description': 'unknown properties specified' + } + LOGGER.error(exception) + return headers_, 400, to_json(exception, self.pretty_print) + else: + select_properties = [] + + LOGGER.debug('processing skipGeometry parameter') + val = args.get('skipGeometry') + if val is not None: + skip_geometry = str2bool(val) + else: + skip_geometry = False + LOGGER.debug('Querying provider') LOGGER.debug('startindex: {}'.format(startindex)) LOGGER.debug('limit: {}'.format(limit)) @@ -888,12 +914,16 @@ class API: LOGGER.debug('sortby: {}'.format(sortby)) LOGGER.debug('bbox: {}'.format(bbox)) LOGGER.debug('datetime: {}'.format(datetime_)) + LOGGER.debug('properties: {}'.format(select_properties)) + LOGGER.debug('skipGeometry: {}'.format(skip_geometry)) try: content = p.query(startindex=startindex, limit=limit, resulttype=resulttype, bbox=bbox, datetime_=datetime_, properties=properties, - sortby=sortby) + sortby=sortby, + select_properties=select_properties, + skip_geometry=skip_geometry) except ProviderConnectionError as err: exception = { 'code': 'NoApplicableCode', diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 87eefbb..0b90dda 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -269,6 +269,32 @@ def get_oas_30(cfg): 'style': 'form', 'explode': False }, + 'properties': { + 'name': 'properties', + 'in': 'query', + 'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'array', + 'items': { + 'type': 'string' + } + } + }, + 'skipGeometry': { + 'name': 'skipGeometry', + 'in': 'query', + 'description': 'This option can be used to skip response geometries for each feature.', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'boolean', + 'default': False + } + }, 'sortby': { 'name': 'sortby', 'in': 'query', @@ -397,6 +423,10 @@ def get_oas_30(cfg): items_path = '{}/items'.format(collection_name_path) + coll_properties = deepcopy(oas['components']['parameters']['properties']) # noqa + + coll_properties['schema']['items']['enum'] = list(p.fields.keys()) + paths[items_path] = { 'get': { 'summary': 'Get {} items'.format(v['title']), @@ -407,6 +437,8 @@ def get_oas_30(cfg): items_f, {'$ref': '{}#/components/parameters/bbox'.format(OPENAPI_YAML['oapif'])}, # noqa {'$ref': '{}#/components/parameters/limit'.format(OPENAPI_YAML['oapif'])}, # noqa + coll_properties, + {'$ref': '#/components/parameters/skipGeometry'}, {'$ref': '#/components/parameters/sortby'}, {'$ref': '#/components/parameters/startindex'} ], diff --git a/pygeoapi/provider/csv_.py b/pygeoapi/provider/csv_.py index b628112..6624829 100644 --- a/pygeoapi/provider/csv_.py +++ b/pygeoapi/provider/csv_.py @@ -72,7 +72,8 @@ class CSVProvider(BaseProvider): return fields def _load(self, startindex=0, limit=10, resulttype='results', - identifier=None, bbox=[], datetime_=None, properties=[]): + identifier=None, bbox=[], datetime_=None, properties=[], + select_properties=[], skip_geometry=False): """ Load CSV data @@ -81,6 +82,8 @@ class CSVProvider(BaseProvider): :param datetime_: temporal (datestamp or extent) :param resulttype: return results or hit limit (default results) :param properties: list of tuples (name, value) + :param select_properties: list of property names + :param skip_geometry: bool of whether to skip geometry (default False) :returns: dict of GeoJSON FeatureCollection """ @@ -103,16 +106,19 @@ class CSVProvider(BaseProvider): for row in itertools.islice(data_, startindex, startindex+limit): feature = {'type': 'Feature'} feature['id'] = row.pop(self.id_field) - feature['geometry'] = { - 'type': 'Point', - 'coordinates': [ - float(row.pop(self.geometry_x)), - float(row.pop(self.geometry_y)) - ] - } - if self.properties: + if not skip_geometry: + feature['geometry'] = { + 'type': 'Point', + 'coordinates': [ + float(row.pop(self.geometry_x)), + float(row.pop(self.geometry_y)) + ] + } + else: + feature['geometry'] = None + if self.properties or select_properties: feature['properties'] = OrderedDict() - for p in self.properties: + for p in set(self.properties) | set(select_properties): try: feature['properties'][p] = row[p] except KeyError as err: @@ -139,7 +145,8 @@ class CSVProvider(BaseProvider): return feature_collection def query(self, startindex=0, limit=10, resulttype='results', - bbox=[], datetime_=None, properties=[], sortby=[]): + bbox=[], datetime_=None, properties=[], sortby=[], + select_properties=[], skip_geometry=False): """ CSV query @@ -150,11 +157,15 @@ class CSVProvider(BaseProvider): :param datetime_: temporal (datestamp or extent) :param properties: list of tuples (name, value) :param sortby: list of dicts (property, order) + :param select_properties: list of property names + :param skip_geometry: bool of whether to skip geometry (default False) :returns: dict of GeoJSON FeatureCollection """ - return self._load(startindex, limit, resulttype) + return self._load(startindex, limit, resulttype, + select_properties=select_properties, + skip_geometry=skip_geometry) def get(self, identifier): """ diff --git a/pygeoapi/provider/elasticsearch_.py b/pygeoapi/provider/elasticsearch_.py index d47d0ae..8671fbd 100644 --- a/pygeoapi/provider/elasticsearch_.py +++ b/pygeoapi/provider/elasticsearch_.py @@ -115,7 +115,8 @@ class ElasticsearchProvider(BaseProvider): return fields_ def query(self, startindex=0, limit=10, resulttype='results', - bbox=[], datetime_=None, properties=[], sortby=[]): + bbox=[], datetime_=None, properties=[], sortby=[], + select_properties=[], skip_geometry=False): """ query Elasticsearch index @@ -126,6 +127,8 @@ class ElasticsearchProvider(BaseProvider): :param datetime_: temporal (datestamp or extent) :param properties: list of tuples (name, value) :param sortby: list of dicts (property, order) + :param select_properties: list of property names + :param skip_geometry: bool of whether to skip geometry (default False) :returns: dict of 0..n GeoJSON features """ @@ -228,15 +231,23 @@ class ElasticsearchProvider(BaseProvider): } query['sort'].append(sort_) - if self.properties: + if self.properties or select_properties: LOGGER.debug('including specified fields: {}'.format( self.properties)) query['_source'] = { - 'includes': list(map(self.mask_prop, self.properties)) + 'includes': list(map(self.mask_prop, + set(self.properties) | set(select_properties))) # noqa } query['_source']['includes'].append(self.mask_prop(self.id_field)) query['_source']['includes'].append('type') query['_source']['includes'].append('geometry') + if skip_geometry: + LOGGER.debug('limiting to specified fields: {}'.format( + select_properties)) + try: + query['_source']['excludes'] = ['geometry'] + except KeyError: + query['_source'] = {'excludes': ['geometry']} try: LOGGER.debug('querying Elasticsearch') LOGGER.debug(json.dumps(query, indent=4)) @@ -352,7 +363,7 @@ class ElasticsearchProvider(BaseProvider): if 'type' not in doc['_source']: feature_['id'] = id_ feature_['type'] = 'Feature' - feature_['geometry'] = doc['_source']['geometry'] + feature_['geometry'] = doc['_source'].get('geometry') feature_['properties'] = {} for key, value in doc['_source'].items(): if key == 'geometry': @@ -363,12 +374,13 @@ class ElasticsearchProvider(BaseProvider): feature_ = doc['_source'] id_ = doc['_source']['properties'][self.id_field] feature_['id'] = id_ + feature_['geometry'] = doc['_source'].get('geometry') if self.properties: feature_thinned = { 'id': id_, 'type': feature_['type'], - 'geometry': feature_['geometry'], + 'geometry': feature_.get('geometry'), 'properties': OrderedDict() } for p in self.properties: diff --git a/pygeoapi/provider/geojson.py b/pygeoapi/provider/geojson.py index 2fc7cb3..644d84a 100644 --- a/pygeoapi/provider/geojson.py +++ b/pygeoapi/provider/geojson.py @@ -84,7 +84,7 @@ class GeoJSONProvider(BaseProvider): fields[f] = 'string' return fields - def _load(self): + def _load(self, skip_geometry=None, select_properties=[]): """Load and validate the source GeoJSON file at self.data @@ -106,10 +106,16 @@ class GeoJSONProvider(BaseProvider): for i in data['features']: if 'id' not in i and self.id_field in i['properties']: i['id'] = i['properties'][self.id_field] + if skip_geometry: + i['geometry'] = None + if self.properties or select_properties: + i['properties'] = {k: v for k, v in i['properties'].items() + if k in set(self.properties) | set(select_properties)} # noqa return data def query(self, startindex=0, limit=10, resulttype='results', - bbox=[], datetime_=None, properties=[], sortby=[]): + bbox=[], datetime_=None, properties=[], sortby=[], + select_properties=[], skip_geometry=False): """ query the provider @@ -120,12 +126,15 @@ class GeoJSONProvider(BaseProvider): :param datetime_: temporal (datestamp or extent) :param properties: list of tuples (name, value) :param sortby: list of dicts (property, order) + :param select_properties: list of property names + :param skip_geometry: bool of whether to skip geometry (default False) :returns: FeatureCollection dict of 0..n GeoJSON features """ # TODO filter by bbox without resorting to third-party libs - data = self._load() + data = self._load(skip_geometry=skip_geometry, + select_properties=select_properties) data['numberMatched'] = len(data['features']) diff --git a/tests/test_api.py b/tests/test_api.py index 39dbf5f..d7e5e58 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -503,6 +503,16 @@ def test_get_collection_items(config, api_): assert code == 200 + rsp_headers, code, response = api_.get_collection_items( + req_headers, {'skipGeometry': 'true'}, 'obs') + + assert json.loads(response)['features'][0]['geometry'] is None + + rsp_headers, code, response = api_.get_collection_items( + req_headers, {'properties': 'foo,bar'}, 'obs') + + assert code == 400 + def test_get_collection_items_json_ld(config, api_): req_headers = make_req_headers() @@ -915,6 +925,6 @@ def test_validate_datetime(): '2001-10-30/2002-10-30') with pytest.raises(ValueError): - _ = validate_datetime(config, '2000/..') + _ = validate_datetime(config, '1999/..') with pytest.raises(ValueError): _ = validate_datetime(config, '../2010') diff --git a/tests/test_csv__provider.py b/tests/test_csv__provider.py index e7538da..cde5fce 100644 --- a/tests/test_csv__provider.py +++ b/tests/test_csv__provider.py @@ -89,6 +89,15 @@ def test_query(config): assert len(results['features'][0]['properties']) == 3 + results = p.query(select_properties=['value']) + assert len(results['features'][0]['properties']) == 1 + + results = p.query(select_properties=['value', 'stn_id']) + assert len(results['features'][0]['properties']) == 2 + + results = p.query(skip_geometry=True) + assert results['features'][0]['geometry'] is None + config['properties'] = ['value', 'stn_id'] p = CSVProvider(config) results = p.query() diff --git a/tests/test_elasticsearch__provider.py b/tests/test_elasticsearch__provider.py index 52fd88b..057fb7a 100644 --- a/tests/test_elasticsearch__provider.py +++ b/tests/test_elasticsearch__provider.py @@ -93,6 +93,15 @@ def test_query(config): assert results['numberMatched'] == 242 assert results['numberReturned'] == 242 + results = p.query(select_properties=['nameascii']) + assert len(results['features'][0]['properties']) == 2 + + results = p.query(select_properties=['nameascii', 'scalerank']) + assert len(results['features'][0]['properties']) == 3 + + results = p.query(skip_geometry=True) + assert results['features'][0]['geometry'] is None + config['properties'] = ['nameascii'] p = ElasticsearchProvider(config) results = p.query() diff --git a/tests/test_geojson_provider.py b/tests/test_geojson_provider.py index 15cfdc6..83c5a30 100644 --- a/tests/test_geojson_provider.py +++ b/tests/test_geojson_provider.py @@ -49,7 +49,11 @@ def fixture(): 'type': 'Point', 'coordinates': [125.6, 10.1]}, 'properties': { - 'name': 'Dinagat Islands'}}]} + 'name': 'Dinagat Islands', + 'foo': 'bar' + }} + ] + } with open(path, 'w') as fh: fh.write(json.dumps(data)) @@ -70,7 +74,7 @@ def test_query(fixture, config): p = GeoJSONProvider(config) fields = p.get_fields() - assert len(fields) == 1 + assert len(fields) == 2 assert fields['name'] == 'string' results = p.query() @@ -79,6 +83,12 @@ def test_query(fixture, config): assert results['numberReturned'] == 1 assert results['features'][0]['id'] == '123-456' + results = p.query(select_properties=['foo']) + assert len(results['features'][0]['properties']) == 1 + + results = p.query(skip_geometry=True) + assert results['features'][0]['geometry'] is None + def test_get(fixture, config): p = GeoJSONProvider(config)