From 8a65087eefce7da8b2b2f52169cb96dfae8f82a3 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sat, 10 Oct 2020 16:02:23 -0400 Subject: [PATCH] [WIP] refactor bbox and datetime for reuse by other APIs (#551) * refactor bbox and datetime for reuse by other APIs * update plugin query arguments * test min/max and temporal --- pygeoapi/api.py | 213 ++++++++++++++++++++++----------- pygeoapi/provider/rasterio_.py | 5 +- pygeoapi/provider/xarray_.py | 5 +- tests/test_api.py | 53 +++++++- 4 files changed, 203 insertions(+), 73 deletions(-) diff --git a/pygeoapi/api.py b/pygeoapi/api.py index b82fac1..ff000e2 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -782,86 +782,31 @@ class API: resulttype = args.get('resulttype') or 'results' LOGGER.debug('Processing bbox parameter') - try: - bbox = args.get('bbox').split(',') - if len(bbox) != 4: + + bbox = args.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: exception = { 'code': 'InvalidParameterValue', - 'description': 'bbox values should be minx,miny,maxx,maxy' + 'description': str(err) } LOGGER.error(exception) return headers_, 400, to_json(exception, self.pretty_print) - except AttributeError: - bbox = [] - try: - bbox = [float(c) for c in bbox] - except ValueError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'bbox values must be numbers' - } - LOGGER.error(exception) - return headers_, 400, to_json(exception, self.pretty_print) LOGGER.debug('Processing datetime parameter') - # TODO: pass datetime to query as a `datetime` object - # we would need to ensure partial dates work accordingly - # as well as setting '..' values to `None` so that underlying - # providers can just assume a `datetime.datetime` object - # - # NOTE: needs testing when passing partials from API to backend datetime_ = args.get('datetime') - datetime_invalid = False - - if (datetime_ is not None and - 'temporal' in collections[dataset]['extents']): - te = collections[dataset]['extents']['temporal'] - - if te['begin'] is not None and te['begin'].tzinfo is None: - te['begin'] = te['begin'].replace(tzinfo=pytz.UTC) - if te['end'] is not None and te['end'].tzinfo is None: - te['end'] = te['end'].replace(tzinfo=pytz.UTC) - - if '/' in datetime_: # envelope - LOGGER.debug('detected time range') - LOGGER.debug('Validating time windows') - datetime_begin, datetime_end = datetime_.split('/') - if datetime_begin != '..': - datetime_begin = dateparse(datetime_begin) - if datetime_begin.tzinfo is None: - datetime_begin = datetime_begin.replace( - tzinfo=pytz.UTC) - - if datetime_end != '..': - datetime_end = dateparse(datetime_end) - if datetime_end.tzinfo is None: - datetime_end = datetime_end.replace(tzinfo=pytz.UTC) - - if te['begin'] is not None and datetime_begin != '..': - if datetime_begin < te['begin']: - datetime_invalid = True - - if te['end'] is not None and datetime_end != '..': - if datetime_end > te['end']: - datetime_invalid = True - - else: # time instant - datetime__ = dateparse(datetime_) - if datetime__ != '..': - if datetime__.tzinfo is None: - datetime__ = datetime__.replace(tzinfo=pytz.UTC) - LOGGER.debug('detected time instant') - if te['begin'] is not None and datetime__ != '..': - if datetime__ < te['begin']: - datetime_invalid = True - if te['end'] is not None and datetime__ != '..': - if datetime__ > te['end']: - datetime_invalid = True - - if datetime_invalid: + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: exception = { 'code': 'InvalidParameterValue', - 'description': 'datetime parameter out of range' + 'description': str(err) } LOGGER.error(exception) return headers_, 400, to_json(exception, self.pretty_print) @@ -941,6 +886,8 @@ class API: LOGGER.debug('limit: {}'.format(limit)) LOGGER.debug('resulttype: {}'.format(resulttype)) LOGGER.debug('sortby: {}'.format(sortby)) + LOGGER.debug('bbox: {}'.format(bbox)) + LOGGER.debug('datetime: {}'.format(datetime_)) try: content = p.query(startindex=startindex, limit=limit, @@ -1269,6 +1216,25 @@ class API: LOGGER.error(exception) return headers_, 500, to_json(exception, self.pretty_print) + LOGGER.debug('Processing bbox parameter') + + bbox = args.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + exception = { + 'code': 'InvalidParameterValue', + 'description': str(err) + } + LOGGER.error(exception) + return headers_, 400, to_json(exception, self.pretty_print) + + query_args['bbox'] = bbox + if 'f' in args: query_args['format_'] = format_ = args['f'] if 'rangeSubset' in args: @@ -2127,3 +2093,110 @@ def check_format(args, headers): format_ = 'json' return format_ + + +def validate_bbox(value=None): + """ + Helper function to validate bbox parameter + + :param bbox: `list` of minx, miny, maxx, maxy + + :returns: `list` of bbox as `float`s + """ + + if value is None: + LOGGER.debug('bbox is empty') + return [] + + bbox = value.split(',') + + if len(bbox) != 4: + msg = 'bbox should be 4 values (minx,miny,maxx,maxy)' + LOGGER.debug(msg) + raise ValueError(msg) + + try: + bbox = [float(c) for c in bbox] + except ValueError as err: + msg = 'bbox values must be numbers' + err.args = (msg,) + LOGGER.debug(msg) + raise + + if bbox[0] > bbox[2] or bbox[1] > bbox[3]: + msg = 'min values should be less than max values' + LOGGER.debug(msg) + raise ValueError(msg) + + return bbox + + +def validate_datetime(resource_def, datetime_=None): + """ + Helper function to validate bbox parameter + + :param resource_def: `dict` of configuration resource definition + :param datetime_: `str` of datetime parameter + + :returns: datetime object(s) + """ + + # TODO: pass datetime to query as a `datetime` object + # we would need to ensure partial dates work accordingly + # as well as setting '..' values to `None` so that underlying + # providers can just assume a `datetime.datetime` object + # + # NOTE: needs testing when passing partials from API to backend + + datetime_invalid = False + + if (datetime_ is not None and 'temporal' in resource_def): + te = resource_def['temporal'] + + if te['begin'] is not None and te['begin'].tzinfo is None: + te['begin'] = te['begin'].replace(tzinfo=pytz.UTC) + if te['end'] is not None and te['end'].tzinfo is None: + te['end'] = te['end'].replace(tzinfo=pytz.UTC) + + if '/' in datetime_: # envelope + LOGGER.debug('detected time range') + LOGGER.debug('Validating time windows') + datetime_begin, datetime_end = datetime_.split('/') + if datetime_begin != '..': + datetime_begin = dateparse(datetime_begin) + if datetime_begin.tzinfo is None: + datetime_begin = datetime_begin.replace( + tzinfo=pytz.UTC) + + if datetime_end != '..': + datetime_end = dateparse(datetime_end) + if datetime_end.tzinfo is None: + datetime_end = datetime_end.replace(tzinfo=pytz.UTC) + + if te['begin'] is not None and datetime_begin != '..': + if datetime_begin < te['begin']: + datetime_invalid = True + + if te['end'] is not None and datetime_end != '..': + if datetime_end > te['end']: + datetime_invalid = True + + else: # time instant + datetime__ = dateparse(datetime_) + if datetime__ != '..': + if datetime__.tzinfo is None: + datetime__ = datetime__.replace(tzinfo=pytz.UTC) + LOGGER.debug('detected time instant') + if te['begin'] is not None and datetime__ != '..': + if datetime__ < te['begin']: + datetime_invalid = True + if te['end'] is not None and datetime__ != '..': + if datetime__ > te['end']: + datetime_invalid = True + + if datetime_invalid: + msg = 'datetime parameter out of range' + LOGGER.debug(msg) + raise ValueError(msg) + + return datetime_ diff --git a/pygeoapi/provider/rasterio_.py b/pygeoapi/provider/rasterio_.py index 00ac50e..bf7ae4b 100644 --- a/pygeoapi/provider/rasterio_.py +++ b/pygeoapi/provider/rasterio_.py @@ -159,11 +159,14 @@ class RasterioProvider(BaseProvider): return rangetype - def query(self, range_subset=[], subsets={}, format_='json'): + def query(self, range_subset=[], subsets={}, bbox=[], datetime=None, + format_='json'): """ Extract data from collection collection :param range_subset: list of bands :param subsets: dict of subset names with lists of ranges + :param bbox: bounding box [minx,miny,maxx,maxy] + :param datetime: temporal (datestamp or extent) :param format_: data format of output :returns: coverage data as dict of CoverageJSON or native format diff --git a/pygeoapi/provider/xarray_.py b/pygeoapi/provider/xarray_.py index 713fa34..d44aa30 100644 --- a/pygeoapi/provider/xarray_.py +++ b/pygeoapi/provider/xarray_.py @@ -180,12 +180,15 @@ class XarrayProvider(BaseProvider): return rangetype - def query(self, range_subset=[], subsets={}, format_='json'): + def query(self, range_subset=[], subsets={}, bbox=[], datetime=None, + format_='json'): """ Extract data from collection collection :param range_subset: list of data variables to return (all if blank) :param subsets: dict of subset names with lists of ranges + :param bbox: bounding box [minx,miny,maxx,maxy] + :param datetime: temporal (datestamp or extent) :param format_: data format of output :returns: coverage data as dict of CoverageJSON or native format diff --git a/tests/test_api.py b/tests/test_api.py index 929953e..b6d0f44 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -36,7 +36,7 @@ import pytest from werkzeug.test import create_environ from werkzeug.wrappers import Request -from pygeoapi.api import API, check_format +from pygeoapi.api import API, check_format, validate_bbox, validate_datetime from pygeoapi.util import yaml_load LOGGER = logging.getLogger(__name__) @@ -857,3 +857,54 @@ def test_check_format(): req_headers = make_req_headers(HTTP_ACCEPT='text/html') args['f'] = 'json' assert check_format(args, req_headers) == 'json' + + +def test_validate_bbox(): + assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] + assert validate_bbox('-142,42,-52,84') == [-142, 42, -52, 84] + assert (validate_bbox('-142.1,42.12,-52.22,84.4') == + [-142.1, 42.12, -52.22, 84.4]) + + with pytest.raises(ValueError): + validate_bbox('1,2,4') + + with pytest.raises(ValueError): + validate_bbox('3,4,1,2') + + +def test_validate_datetime(): + config = yaml_load(''' + temporal: + begin: 2000-10-30T18:24:39Z + end: 2007-10-30T08:57:29Z + ''') + + # test time instant + assert validate_datetime(config, '2004') == '2004' + assert validate_datetime(config, '2004-10') == '2004-10' + assert validate_datetime(config, '2001-10-30') == '2001-10-30' + + with pytest.raises(ValueError): + _ = validate_datetime(config, '2009-10-30') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2000-09-09') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2000-10-30T17:24:39Z') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2007-10-30T08:58:29Z') + + # test time envelope + assert validate_datetime(config, '2004/2005') == '2004/2005' + assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' + assert (validate_datetime(config, '2001-10-30/2002-10-30') == + '2001-10-30/2002-10-30') + assert validate_datetime(config, '2004/..') == '2004/..' + assert validate_datetime(config, '../2005') == '../2005' + assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' + assert (validate_datetime(config, '2001-10-30/2002-10-30') == + '2001-10-30/2002-10-30') + + with pytest.raises(ValueError): + _ = validate_datetime(config, '2000/..') + with pytest.raises(ValueError): + _ = validate_datetime(config, '../2010')