huanld/pygeoapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Time Querying in EDR Provider (#1247)

Browse Source 
* update time handling for edr provider

* add logging for time parsing

* fix time handling errors

Convert times to np.datetime64 to allow greater than/less than comparison. If slicing over time, separate xarray sel into temporal and spatial component (cannot use method='nearest' for time slices).

* fix temporal query dictionary

* fix time error handling

* fix list handling for single timestep

* single timestep handling

* spatial-only query amendment

* code clean-up

* convert datetime

* move code to methods

for reuse between position and cube. #1239

* formatting fixes

#1239

* time query test

#1239

* clean-up

* move time querying tests to api testing

* Update test_api.py

* update time key

* modify test values

* updates for desired query behavior

#1239

* Update xarray_edr.py

* sanitize datetime_ parameter with Z

OpenAPI document provides RFC 3339 compliant examples (exclusively specify UTC as the timezone with a trailing Z; want to avoid confusion between allowed datetime query and the documentation. #1239

* requested changes from @webb-ben and @tomkralidis

#1247
This commit is contained in:
Sarah Jordan
2023-05-17 20:19:10 -05:00
committed by GitHub
parent dea6ffc48e
commit dcfdfdf799
2 changed files with 138 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pygeoapi/provider/xarray_edr.py
+81 -19
View File
@@ -29,6 +29,8 @@
import logging
import numpy as np
from pygeoapi.provider.base import ProviderNoDataError, ProviderQueryError
from pygeoapi.provider.base_edr import BaseEDRProvider
from pygeoapi.provider.xarray_ import _to_datetime_string, XarrayProvider
@@ -106,7 +108,7 @@ class XarrayEDRProvider(BaseEDRProvider, XarrayProvider):
datetime_ = kwargs.get('datetime_')
if datetime_ is not None:
query_params[self._coverage_properties['time_axis_label']] = datetime_ # noqa
query_params[self.time_field] = self._make_datetime(datetime_)
LOGGER.debug(f'query parameters: {query_params}')
@@ -116,13 +118,22 @@ class XarrayEDRProvider(BaseEDRProvider, XarrayProvider):
data = self._data[[*select_properties]]
else:
data = self._data
data = data.sel(query_params, method='nearest')
if (datetime_ is not None and
isinstance(query_params[self.time_field], slice)): # noqa
# separate query into spatial and temporal components
LOGGER.debug('Separating temporal query')
time_query = {self.time_field:
query_params[self.time_field]}
remaining_query = {key: val for key,
val in query_params.items()
if key != self.time_field}
data = data.sel(time_query).sel(remaining_query,
method='nearest')
else:
data = data.sel(query_params, method='nearest')
except KeyError:
raise ProviderNoDataError()
if len(data.coords[self.time_field].values) < 1:
raise ProviderNoDataError()
try:
height = data.dims[self.y_field]
except KeyError:
@@ -131,18 +142,16 @@ class XarrayEDRProvider(BaseEDRProvider, XarrayProvider):
width = data.dims[self.x_field]
except KeyError:
width = 1
time, time_steps = self._parse_time_metadata(data, kwargs)
bbox = wkt.bounds
out_meta = {
'bbox': [bbox[0], bbox[1], bbox[2], bbox[3]],
"time": [
_to_datetime_string(data.coords[self.time_field].values[0]),
_to_datetime_string(data.coords[self.time_field].values[-1])
],
"time": time,
"driver": "xarray",
"height": height,
"width": width,
"time_steps": data.dims[self.time_field],
"time_steps": time_steps,
"variables": {var_name: var.attrs
for var_name, var in data.variables.items()}
}
@@ -187,7 +196,7 @@ class XarrayEDRProvider(BaseEDRProvider, XarrayProvider):
datetime_ = kwargs.get('datetime_')
if datetime_ is not None:
query_params[self._coverage_properties['time_axis_label']] = datetime_ # noqa
query_params[self.time_field] = self._make_datetime(datetime_)
LOGGER.debug(f'query parameters: {query_params}')
try:
@@ -200,11 +209,9 @@ class XarrayEDRProvider(BaseEDRProvider, XarrayProvider):
except KeyError:
raise ProviderNoDataError()
if len(data.coords[self.time_field].values) < 1:
raise ProviderNoDataError()
height = data.dims[self.y_field]
width = data.dims[self.x_field]
time, time_steps = self._parse_time_metadata(data, kwargs)
out_meta = {
'bbox': [
@@ -213,16 +220,71 @@ class XarrayEDRProvider(BaseEDRProvider, XarrayProvider):
data.coords[self.x_field].values[-1],
data.coords[self.y_field].values[-1]
],
"time": [
_to_datetime_string(data.coords[self.time_field].values[0]),
_to_datetime_string(data.coords[self.time_field].values[-1])
],
"time": time,
"driver": "xarray",
"height": height,
"width": width,
"time_steps": data.dims[self.time_field],
"time_steps": time_steps,
"variables": {var_name: var.attrs
for var_name, var in data.variables.items()}
}
return self.gen_covjson(out_meta, data, self.fields)
def _make_datetime(self, datetime_):
"""
Make xarray datetime query
:param datetime_: temporal (datestamp or extent)
:returns: xarray datetime query
"""
datetime_ = datetime_.rstrip('Z').replace('Z/', '/')
if '/' in datetime_:
begin, end = datetime_.split('/')
if begin == '..':
begin = self._data[self.time_field].min().values
if end == '..':
end = self._data[self.time_field].max().values
if np.datetime64(begin) < np.datetime64(end):
return slice(begin, end)
else:
LOGGER.debug('Reversing slicing from high to low')
return slice(end, begin)
else:
return datetime_
def _get_time_range(self, data):
"""
Make xarray dataset temporal extent
:param data: xarray dataset
:returns: list of temporal extent
"""
time = data.coords[self.time_field]
if time.size == 0:
raise ProviderNoDataError()
else:
start = _to_datetime_string(data[self.time_field].values.min())
end = _to_datetime_string(data[self.time_field].values.max())
return [start, end]
def _parse_time_metadata(self, data, kwargs):
"""
Parse time information for output metadata.
:param data: xarray dataset
:param kwargs: dictionary
:returns: list of temporal extent, number of timesteps
"""
try:
time = self._get_time_range(data)
except KeyError:
time = []
try:
time_steps = data.coords[self.time_field].size
except KeyError:
time_steps = kwargs.get('limit')
return time, time_steps
tests/test_api.py
+57
View File
@@ -1933,6 +1933,63 @@ def test_get_collection_edr_query(config, api_):
assert len(data['parameters'].keys()) == 1
assert list(data['parameters'].keys())[0] == 'SST'
# Zulu time zone
req = mock_request({
'coords': 'POINT(11 11)',
'datetime': '2000-01-17T00:00:00Z/2000-06-16T23:00:00Z'
})
rsp_headers, code, response = api_.get_collection_edr_query(
req, 'icoads-sst', None, 'position')
assert code == HTTPStatus.OK
# bounded date range
req = mock_request({
'coords': 'POINT(11 11)',
'datetime': '2000-01-17/2000-06-16'
})
rsp_headers, code, response = api_.get_collection_edr_query(
req, 'icoads-sst', None, 'position')
assert code == HTTPStatus.OK
data = json.loads(response)
time_dict = data['domain']['axes']['TIME']
assert time_dict['start'] == '2000-02-15T16:29:05.999999999'
assert time_dict['stop'] == '2000-06-16T10:25:30.000000000'
assert time_dict['num'] == 5
# unbounded date range - start
req = mock_request({
'coords': 'POINT(11 11)',
'datetime': '../2000-06-16'
})
rsp_headers, code, response = api_.get_collection_edr_query(
req, 'icoads-sst', None, 'position')
assert code == HTTPStatus.OK
data = json.loads(response)
time_dict = data['domain']['axes']['TIME']
assert time_dict['start'] == '2000-01-16T06:00:00.000000000'
assert time_dict['stop'] == '2000-06-16T10:25:30.000000000'
assert time_dict['num'] == 6
# unbounded date range - end
req = mock_request({
'coords': 'POINT(11 11)',
'datetime': '2000-06-16/..'
})
rsp_headers, code, response = api_.get_collection_edr_query(
req, 'icoads-sst', None, 'position')
assert code == HTTPStatus.OK
data = json.loads(response)
time_dict = data['domain']['axes']['TIME']
assert time_dict['start'] == '2000-06-16T10:25:30.000000000'
assert time_dict['stop'] == '2000-12-16T01:20:05.999999996'
assert time_dict['num'] == 7
# some data
req = mock_request({
'coords': 'POINT(11 11)', 'datetime': '2000-01-16'