From 951a1fb48625ffa0e18d327fe3d34be512fe25b1 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 15 Mar 2021 11:37:40 -0400 Subject: [PATCH] implement OGC EDR API (#658) * implement OGC EDR API * add docs/tests * fix tests --- Dockerfile | 2 +- docs/source/configuration.rst | 2 +- docs/source/data-publishing/index.rst | 1 + .../data-publishing/ogcapi-coverages.rst | 2 +- docs/source/data-publishing/ogcapi-edr.rst | 84 ++++++++ docs/source/introduction.rst | 9 +- docs/source/tour.rst | 20 ++ pygeoapi-config.yml | 26 +++ pygeoapi/api.py | 193 +++++++++++++++++- pygeoapi/flask_app.py | 33 +++ pygeoapi/openapi.py | 53 ++++- pygeoapi/plugin.py | 3 +- pygeoapi/provider/filesystem.py | 2 - pygeoapi/provider/xarray_.py | 17 +- pygeoapi/provider/xarray_edr.py | 166 +++++++++++++++ pygeoapi/starlette_app.py | 40 ++++ pygeoapi/templates/collections/edr/query.html | 80 ++++++++ requirements.txt | 1 + tests/pygeoapi-test-config.yml | 26 +++ tests/test_api.py | 90 +++++++- tests/test_util.py | 2 +- 21 files changed, 830 insertions(+), 22 deletions(-) create mode 100644 docs/source/data-publishing/ogcapi-edr.rst create mode 100644 pygeoapi/provider/xarray_edr.py create mode 100644 pygeoapi/templates/collections/edr/query.html diff --git a/Dockerfile b/Dockerfile index 497a21d..4089e51 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,7 +68,7 @@ ARG ADD_DEB_PACKAGES="python3-gdal python3-psycopg2 python3-xarray python3-scipy ENV TZ=${TIMEZONE} \ DEBIAN_FRONTEND="noninteractive" \ DEB_BUILD_DEPS="software-properties-common curl unzip" \ - DEB_PACKAGES="python3-pip python3-setuptools python3-distutils python3-yaml python3-dateutil python3-tz python3-flask python3-flask-cors python3-unicodecsv python3-click python3-greenlet python3-gevent python3-wheel gunicorn libsqlite3-mod-spatialite ${ADD_DEB_PACKAGES}" + DEB_PACKAGES="python3-pip python3-setuptools python3-distutils python3-shapely python3-yaml python3-dateutil python3-tz python3-flask python3-flask-cors python3-unicodecsv python3-click python3-greenlet python3-gevent python3-wheel gunicorn libsqlite3-mod-spatialite ${ADD_DEB_PACKAGES}" RUN mkdir -p /pygeoapi/pygeoapi # Add files required for pip/setuptools diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index c0c62d2..bb07f1b 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -170,7 +170,7 @@ default. # see pygeoapi.plugin for supported providers # for custom built plugins, use the import path (e.g. mypackage.provider.MyProvider) # see Plugins section for more information - - type: feature # underlying data geospatial type: (allowed values are: feature, coverage, record, tile) + - type: feature # underlying data geospatial type: (allowed values are: feature, coverage, record, tile, edr) default: true # optional: if not specified, the first provider definition is considered the default name: CSV data: tests/data/obs.csv # required: the data filesystem path or URL, depending on plugin setup diff --git a/docs/source/data-publishing/index.rst b/docs/source/data-publishing/index.rst index 31ea103..feb64b5 100644 --- a/docs/source/data-publishing/index.rst +++ b/docs/source/data-publishing/index.rst @@ -23,4 +23,5 @@ return back data to the pygeoapi API framework in a plug and play fashion. ogcapi-tiles ogcapi-processes ogcapi-records + ogcapi-edr stac diff --git a/docs/source/data-publishing/ogcapi-coverages.rst b/docs/source/data-publishing/ogcapi-coverages.rst index b2df387..07f8061 100644 --- a/docs/source/data-publishing/ogcapi-coverages.rst +++ b/docs/source/data-publishing/ogcapi-coverages.rst @@ -79,7 +79,7 @@ The `xarray`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data. .. note:: `Zarr`_ files are directories with files and subdirectories. Therefore - a zip file is returned upon request for said format. + a zip file is returned upon request for said format. Data access examples -------------------- diff --git a/docs/source/data-publishing/ogcapi-edr.rst b/docs/source/data-publishing/ogcapi-edr.rst new file mode 100644 index 0000000..334b715 --- /dev/null +++ b/docs/source/data-publishing/ogcapi-edr.rst @@ -0,0 +1,84 @@ +.. _ogcapi-edr: + +Publishing data to OGC API - Environmental Data Retrieval +========================================================= + +The `OGC Environmental Data Retrieval (EDR) (API)`_ provides a family of +lightweight query interfaces to access spatio-temporal data resources. + +To add spatio-temporal data to pygeoapi for EDR query interfaces, you +can use the dataset example in :ref:`configuration` as a baseline and +modify accordingly. + +Providers +--------- + +pygeoapi core EDR providers are listed below, along with a matrix of supported query +parameters. + +.. csv-table:: + :header: Provider, coords, parameter-name, datetime + :align: left + + xarray-edr,✅,✅,✅ + + +Below are specific connection examples based on supported providers. + +Connection examples +------------------- + +xarray-edr +^^^^^^^^^^ + +The `xarray-edr`_ provider plugin reads and extracts `NetCDF`_ and `Zarr`_ data via `xarray`_. + +.. code-block:: yaml + + providers: + - type: edr + name: xarray-edr + data: tests/data/coads_sst.nc + # optionally specify x/y/time fields, else provider will attempt + # to derive automagically + x_field: lat + x_field: lon + time_field: time + format: + name: netcdf + mimetype: application/x-netcdf + + providers: + - type: edr + name: xarray-edr + data: tests/data/analysed_sst.zarr + format: + name: zarr + mimetype: application/zip + +.. note:: + + `Zarr`_ files are directories with files and subdirectories. Therefore + a zip file is returned upon request for said format. + +Data access examples +-------------------- + +- list all collections + - http://localhost:5000/collections +- overview of dataset + - http://localhost:5000/collections/foo +- dataset position query + - http://localhost:5000/collections/foo/position?coords=POINT(-75%2045) +- dataset position query for a specific parameter + - http://localhost:5000/collections/foo/position?coords=POINT(-75%2045)¶meter-name=SST +- dataset position query for a specific parameter and time step + - http://localhost:5000/collections/foo/position?coords=POINT(-75%2045)¶meter-name=SST&datetime=2000-01-16 + + +.. _`xarray`: https://xarray.pydata.org +.. _`NetCDF`: https://en.wikipedia.org/wiki/NetCDF +.. _`Zarr`: https://zarr.readthedocs.io/en/stable + + +.. _`OGC Environmental Data Retrieval (EDR) (API)`: https://github.com/opengeospatial/ogcapi-coverages diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index b16f6be..9d07c22 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -10,7 +10,12 @@ Features - out of the box modern OGC API server - certified OGC Compliant and Reference Implementation for OGC API - Features -- additionally implements OGC API - Coverages, OGC API - Tiles, OGC API - Processes, OGC API - Records and SpatioTemporal Asset Library +- additionally implements + - OGC API - Coverages + - OGC API - Tiles + - OGC API - Processes + - OGC API - Environmental Data Retrieval + - SpatioTemporal Asset Library - out of the box data provider plugins for rasterio, GDAL/OGR, Elasticsearch, PostgreSQL/PostGIS - easy to use OpenAPI / Swagger documentation for developers - supports JSON, GeoJSON, HTML and CSV output @@ -41,6 +46,7 @@ Standards are at the core of pygeoapi. Below is the project's standards support `OGC API - Tiles`_,Implementing `OGC API - Processes`_,Implementing `OGC API - Records`_,Implementing + `OGC API - Environmental Data Retrieval`_,Implementing `SpatioTemporal Asset Catalog`_,Implementing @@ -51,4 +57,5 @@ Standards are at the core of pygeoapi. Below is the project's standards support .. _`OGC API - Tiles`: https://github.com/opengeospatial/ogcapi-tiles .. _`OGC API - Processes`: https://github.com/opengeospatial/ogcapi-processes .. _`OGC API - Records`: https://github.com/opengeospatial/ogcapi-records +.. _`OGC API - Environmental Data Retrieval`: https://github.com/opengeospatial/ogcapi-environmental-data-retrieval .. _`SpatioTemporal Asset Catalog`: https://stacspec.org diff --git a/docs/source/tour.rst b/docs/source/tour.rst index 78b24c3..c1be440 100644 --- a/docs/source/tour.rst +++ b/docs/source/tour.rst @@ -148,6 +148,7 @@ This page provides metadata catalogue search capabilities .. seealso:: :ref:`ogcapi-records` for more OGC API - Records request examples. + Processes --------- @@ -160,6 +161,25 @@ The processes page provides a list of process integrated onto the server, along :ref:`ogcapi-processes` for more OGC API - Processes request examples. +Environmental data retrieval +---------------------------- + +http://localhost:5000/collections/edr-test + +This page provides, in addition to a common collection description, specific +link relations for EDR queries if the collection has an EDR capability, as +well as supported parameter names to select. + +http://localhost:5000/collections/edr-test/position?coords=POINT(111 13)¶meter-name=SST&f=json + +This page executes a position query against a given parameter name, providing +a response in CoverageJSON. + + +.. seealso:: + :ref:`ogcapi-edr` for more OGC API - EDR request examples. + + SpatioTemporal Assets --------------------- diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml index fb6a5d1..9a9f4bc 100644 --- a/pygeoapi-config.yml +++ b/pygeoapi-config.yml @@ -180,6 +180,32 @@ resources: name: GRIB2 mimetype: application/x-grib2 + icoads-sst: + type: collection + title: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + description: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + keywords: + - icoads + - sst + - air temperature + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + links: + - type: text/html + rel: canonical + title: information + href: https://psl.noaa.gov/data/gridded/data.coads.1deg.html + hreflang: en-US + providers: + - type: edr + name: xarray-edr + data: tests/data/coads_sst.nc + format: + name: NetCDF + mimetype: application/x-netcdf + test-data: type: stac-collection title: pygeoapi test data diff --git a/pygeoapi/api.py b/pygeoapi/api.py index e32fe76..d4da0b4 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -43,6 +43,8 @@ import urllib.parse from copy import deepcopy from dateutil.parser import parse as dateparse +from shapely.wkt import loads as shapely_loads +from shapely.errors import WKTReadingError import pytz from pygeoapi import __version__ @@ -79,6 +81,8 @@ HEADERS = { FORMATS = ['json', 'html', 'jsonld'] CONFORMANCE = [ + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections', 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30', 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html', @@ -88,13 +92,12 @@ CONFORMANCE = [ 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/html', 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/req/core', 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/req/collections', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections', 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/core', 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/sorting', 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/opensearch', 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/html' + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-edr-1/1.0/conf/core' ] OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' @@ -532,14 +535,15 @@ class API: p = load_plugin('provider', get_provider_by_type( self.config['resources'][k]['providers'], 'coverage')) + collection['crs'] = [p.crs] + collection['domainset'] = p.get_coverage_domainset() + collection['rangetype'] = p.get_coverage_rangetype() except ProviderConnectionError: msg = 'connection error (check logs)' return self.get_exception( 500, headers_, format_, 'NoApplicableCode', msg) - - collection['crs'] = [p.crs] - collection['domainset'] = p.get_coverage_domainset() - collection['rangetype'] = p.get_coverage_rangetype() + except ProviderTypeError: + pass try: tile = get_provider_by_type(v['providers'], 'tile') @@ -563,6 +567,45 @@ class API: self.config['server']['url'], k) }) + try: + edr = get_provider_by_type(v['providers'], 'edr') + except ProviderTypeError: + edr = None + + if edr and dataset is not None: + LOGGER.debug('Adding EDR links') + try: + p = load_plugin('provider', get_provider_by_type( + self.config['resources'][dataset]['providers'], + 'edr')) + parameters = p.get_fields() + if parameters: + collection['parameters'] = {} + for f in parameters['field']: + collection['parameters'][f['id']] = f + + for qt in p.get_query_types(): + collection['links'].append({ + 'type': 'text/json', + 'rel': 'data', + 'title': '{} query for this collection as JSON'.format(qt), # noqa + 'href': '{}/collections/{}/{}?f=json'.format( + self.config['server']['url'], k, qt) + }) + collection['links'].append({ + 'type': 'text/html', + 'rel': 'data', + 'title': '{} query for this collection as HTML'.format(qt), # noqa + 'href': '{}/collections/{}/{}?f=html'.format( + self.config['server']['url'], k, qt) + }) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return self.get_exception( + 500, headers_, format_, 'NoApplicableCode', msg) + except ProviderTypeError: + pass + if dataset is not None and k == dataset: fcm = collection break @@ -2115,6 +2158,142 @@ tiles/{{{}}}/{{{}}}/{{{}}}/{{{}}}?f=mvt' LOGGER.info(response) return {}, http_status, response + def get_collection_edr_query(self, headers, args, dataset, instance, + query_type): + """ + Queries collection EDR + :param headers: dict of HTTP headers + :param args: dict of HTTP request parameters + :param dataset: dataset name + :param dataset: instance name + :param query_type: EDR query type + :returns: tuple of headers, status code, content + """ + + headers_ = HEADERS.copy() + + query_args = {} + formats = FORMATS + formats.extend(f.lower() for f in PLUGINS['formatter'].keys()) + + collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'collection') + + format_ = check_format(args, headers) + + if dataset not in collections.keys(): + msg = 'Invalid collection' + return self.get_exception( + 400, headers_, format_, 'InvalidParameterValue', msg) + + if format_ is not None and format_ not in formats: + msg = 'Invalid format' + return self.get_exception( + 400, headers_, format_, 'InvalidParameterValue', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing datetime parameter') + datetime_ = args.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return self.get_exception( + 400, headers_, format_, 'InvalidParameterValue', msg) + + LOGGER.debug('Processing parameter-name parameter') + parameternames = args.get('parameter-name', []) + if parameternames: + parameternames = parameternames.split(',') + + LOGGER.debug('Processing coords parameter') + wkt = args.get('coords', None) + + if wkt is None: + msg = 'missing coords parameter' + return self.get_exception( + 400, headers_, format_, 'InvalidParameterValue', msg) + + try: + wkt = shapely_loads(wkt) + except WKTReadingError: + msg = 'invalid coords parameter' + return self.get_exception( + 400, headers_, format_, 'InvalidParameterValue', msg) + + LOGGER.debug('Processing z parameter') + z = args.get('z') + + LOGGER.debug('Loading provider') + try: + p = load_plugin('provider', get_provider_by_type( + collections[dataset]['providers'], 'edr')) + except ProviderTypeError: + msg = 'invalid provider type' + return self.get_exception( + 500, headers_, format_, 'NoApplicableCode', msg) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return self.get_exception( + 500, headers_, format_, 'NoApplicableCode', msg) + except ProviderQueryError: + msg = 'query error (check logs)' + return self.get_exception( + 500, headers_, format_, 'NoApplicableCode', msg) + + if instance is not None and not p.get_instance(instance): + msg = 'Invalid instance identifier' + return self.get_exception( + 400, headers_, format_, 'InvalidParameterValue', msg) + + if query_type not in p.get_query_types(): + msg = 'Unsupported query type' + return self.get_exception( + 400, headers_, format_, 'InvalidParameterValue', msg) + + parametername_matches = list( + filter( + lambda p: p['id'] in parameternames, p.get_fields()['field'] + ) + ) + + if len(parametername_matches) < len(parameternames): + msg = 'Invalid parameter-name' + return self.get_exception( + 400, headers_, format_, 'InvalidParameterValue', msg) + + query_args = dict( + query_type=query_type, + instance=instance, + format_=format_, + datetime_=datetime_, + select_properties=parameternames, + wkt=wkt, + z=z + ) + + try: + data = p.query(**query_args) + except ProviderNoDataError: + msg = 'No data found' + return self.get_exception( + 204, headers_, format_, 'NoMatch', msg) + except ProviderQueryError: + msg = 'query error (check logs)' + return self.get_exception( + 500, headers_, format_, 'NoApplicableCode', msg) + + if format_ == 'html': # render + headers_['Content-Type'] = 'text/html' + content = render_j2_template( + self.config, 'collections/edr/query.html', data) + else: + content = to_json(data, self.pretty_print) + + return headers_, 200, content + @pre_process @jsonldify def get_stac_root(self, headers_, format_): diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index b4acf8b..86acb4c 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -465,6 +465,39 @@ def get_process_job_result_resource(process_id, job_id, resource): return response +@BLUEPRINT.route('/collections//position') +@BLUEPRINT.route('/collections//area') +@BLUEPRINT.route('/collections//cube') +@BLUEPRINT.route('/collections//trajectory') +@BLUEPRINT.route('/collections//corridor') +@BLUEPRINT.route('/collections//instances//position') # noqa +@BLUEPRINT.route('/collections//instances//area') +@BLUEPRINT.route('/collections//instances//cube') +@BLUEPRINT.route('/collections//instances//trajectory') # noqa +@BLUEPRINT.route('/collections//instances//corridor') # noqa +def get_collection_edr_query(collection_id, instance_id=None): + """ + OGC EDR API endpoints + + :param collection_id: collection identifier + :param instance_id: instance identifier + + :returns: HTTP response + """ + + query_type = request.path.split('/')[-1] + + headers, status_code, content = api_.get_collection_edr_query( + request.headers, request.args, collection_id, instance_id, query_type) + + response = make_response(content, status_code) + + if headers: + response.headers = headers + + return response + + @BLUEPRINT.route('/stac') def stac_catalog_root(): """ diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 5d271cd..1c968ac 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -48,7 +48,8 @@ OPENAPI_YAML = { 'oacov': 'https://raw.githubusercontent.com/tomkralidis/ogcapi-coverages-1/fix-cis/yaml-unresolved', # noqa 'oapit': 'https://raw.githubusercontent.com/opengeospatial/ogcapi-tiles/master/openapi/swaggerhub/tiles.yaml', # noqa 'oapimt': 'https://raw.githubusercontent.com/opengeospatial/ogcapi-tiles/master/openapi/swaggerhub/map-tiles.yaml', # noqa - 'oapir': 'https://raw.githubusercontent.com/opengeospatial/ogcapi-records/master/core/openapi' # noqa + 'oapir': 'https://raw.githubusercontent.com/opengeospatial/ogcapi-records/master/core/openapi', # noqa + 'oaedr': 'https://raw.githubusercontent.com/opengeospatial/ogcapi-environmental-data-retrieval/master/candidate-standard/openapi', # noqa } @@ -726,6 +727,56 @@ def get_oas_30(cfg): } } + LOGGER.debug('setting up tiles endpoints') + edr_extension = filter_providers_by_type( + collections[k]['providers'], 'edr') + + if edr_extension: + ep = load_plugin('provider', edr_extension) + + edr_query_endpoints = [] + + for qt in ep.get_query_types(): + edr_query_endpoints.append({ + 'path': '{}/{}'.format(collection_name_path, qt), + 'qt': qt, + 'op_id': 'query{}{}'.format(qt.capitalize(), k.capitalize()) # noqa + }) + if ep.instances: + edr_query_endpoints.append({ + 'path': '{}/instances/{{instanceId}}/{}'.format(collection_name_path, qt), # noqa + 'qt': qt, + 'op_id': 'query{}Instance{}'.format(qt.capitalize(), k.capitalize()) # noqa + }) + + for eqe in edr_query_endpoints: + paths[eqe['path']] = { + 'get': { + 'summary': 'query {} by {}'.format(v['description'], eqe['qt']), # noqa + 'description': v['description'], + 'tags': [k], + 'operationId': eqe['op_id'], + 'parameters': [ + {'$ref': '{}/parameters/{}Coords.yaml'.format(OPENAPI_YAML['oaedr'], eqe['qt'])}, # noqa + {'$ref': '{}#/components/parameters/datetime'.format(OPENAPI_YAML['oapif'])}, # noqa + {'$ref': '{}/parameters/parameter-name.yaml'.format(OPENAPI_YAML['oaedr'])}, # noqa + {'$ref': '{}/parameters/z.yaml'.format(OPENAPI_YAML['oaedr'])}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/prs.coverage+json': { + 'schema': { + '$ref': '{}/schemas/coverageJSON.yaml'.format(OPENAPI_YAML['oaedr'])} # noqa + } + } + } + } + } + } + LOGGER.debug('setting up STAC') stac_collections = filter_dict_by_key_value(cfg['resources'], 'type', 'stac-collection') diff --git a/pygeoapi/plugin.py b/pygeoapi/plugin.py index c2206c1..ef42e67 100644 --- a/pygeoapi/plugin.py +++ b/pygeoapi/plugin.py @@ -49,7 +49,8 @@ PLUGINS = { 'rasterio': 'pygeoapi.provider.rasterio_.RasterioProvider', 'xarray': 'pygeoapi.provider.xarray_.XarrayProvider', 'MVT': 'pygeoapi.provider.mvt.MVTProvider', - 'TinyDBCatalogue': 'pygeoapi.provider.tinydb_.TinyDBCatalogueProvider' + 'TinyDBCatalogue': 'pygeoapi.provider.tinydb_.TinyDBCatalogueProvider', + 'xarray-edr': 'pygeoapi.provider.xarray_edr.XarrayEDRProvider' }, 'formatter': { 'CSV': 'pygeoapi.formatter.csv_.CSVFormatter' diff --git a/pygeoapi/provider/filesystem.py b/pygeoapi/provider/filesystem.py index 12fc4e3..c35e467 100644 --- a/pygeoapi/provider/filesystem.py +++ b/pygeoapi/provider/filesystem.py @@ -70,8 +70,6 @@ class FileSystemProvider(BaseProvider): thispath = os.path.join(baseurl, urlpath) - print("THISPATH", thispath) - resource_type = None root_link = None child_links = [] diff --git a/pygeoapi/provider/xarray_.py b/pygeoapi/provider/xarray_.py index f7b73c7..ec85904 100644 --- a/pygeoapi/provider/xarray_.py +++ b/pygeoapi/provider/xarray_.py @@ -57,7 +57,7 @@ class XarrayProvider(BaseProvider): super().__init__(provider_def) try: - if provider_def['format']['name'] == 'zarr': + if provider_def['data'].endswith('.zarr'): open_func = xarray.open_zarr else: open_func = xarray.open_dataset @@ -312,10 +312,19 @@ class XarrayProvider(BaseProvider): minx, miny, maxx, maxy = metadata['bbox'] mint, maxt = metadata['time'] - if data.coords[self.y_field].values[0] > data.coords[self.y_field].values[-1]: # noqa + try: + tmp_min = data.coords[self.y_field].values[0] + except IndexError: + tmp_min = data.coords[self.y_field].values + try: + tmp_max = data.coords[self.y_field].values[-1] + except IndexError: + tmp_max = data.coords[self.y_field].values + + if tmp_min > tmp_max: LOGGER.debug('Reversing direction of {}'.format(self.y_field)) - miny = data.coords[self.y_field].values[-1] - maxy = data.coords[self.y_field].values[0] + miny = tmp_max + maxy = tmp_min cj = { 'type': 'Coverage', diff --git a/pygeoapi/provider/xarray_edr.py b/pygeoapi/provider/xarray_edr.py new file mode 100644 index 0000000..d8e5d0c --- /dev/null +++ b/pygeoapi/provider/xarray_edr.py @@ -0,0 +1,166 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2020 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging + +from pygeoapi.provider.base import ProviderNoDataError +from pygeoapi.provider.xarray_ import _to_datetime_string, XarrayProvider + +LOGGER = logging.getLogger(__name__) + + +class XarrayEDRProvider(XarrayProvider): + """EDR Provider""" + + def __init__(self, provider_def): + """ + Initialize object + + :param provider_def: provider definition + + :returns: pygeoapi.provider.rasterio_.RasterioProvider + """ + + XarrayProvider.__init__(self, provider_def) + self.instances = [] + + def get_fields(self): + """ + Get provider field information (names, types) + + :returns: dict of dicts of parameters + """ + + return self.get_coverage_rangetype() + + def get_instance(self, instance): + """ + Validate instance identifier + + :returns: `bool` of whether instance is valid + """ + + return NotImplementedError() + + def get_query_types(self): + """ + Provide supported query types + + :returns: list of EDR query types + """ + + return ['position'] + + def query(self, **kwargs): + """ + Extract data from collection collection + + :param query_type: query type + :param wkt: `shapely.geometry` WKT geometry + :param datetime_: temporal (datestamp or extent) + :param select_properties: list of parameters + :param z: vertical level(s) + :param format_: data format of output + + :returns: coverage data as dict of CoverageJSON or native format + """ + + query_params = {} + + LOGGER.debug('Query parameters: {}'.format(kwargs)) + + LOGGER.debug('Query type: {}'.format(kwargs.get('query_type'))) + + wkt = kwargs.get('wkt') + if wkt is not None: + LOGGER.debug('Processing WKT') + LOGGER.debug('Geometry type: {}'.format(wkt.type)) + if wkt.type == 'Point': + query_params[self._coverage_properties['x_axis_label']] = wkt.x + query_params[self._coverage_properties['y_axis_label']] = wkt.y + elif wkt.type == 'LineString': + query_params[self._coverage_properties['x_axis_label']] = wkt.xy[0] # noqa + query_params[self._coverage_properties['y_axis_label']] = wkt.xy[1] # noqa + elif wkt.type == 'Polygon': + query_params[self._coverage_properties['x_axis_label']] = slice(wkt.bounds[0], wkt.bounds[2]) # noqa + query_params[self._coverage_properties['y_axis_label']] = slice(wkt.bounds[1], wkt.bounds[3]) # noqa + pass + + LOGGER.debug('Processing parameter-name') + select_properties = kwargs.get('select_properties') + + # example of fetching instance passed + # TODO: apply accordingly + instance = kwargs.get('instance') + LOGGER.debug('instance: {}'.format(instance)) + + datetime_ = kwargs.get('datetime_') + if datetime_ is not None: + query_params[self._coverage_properties['time_axis_label']] = datetime_ # noqa + + LOGGER.debug('query parameters: {}'.format(query_params)) + + try: + if select_properties: + self.fields = select_properties + data = self._data[[*select_properties]] + else: + data = self._data + data = data.sel(query_params, method='nearest') + except KeyError: + raise ProviderNoDataError() + + if len(data.coords[self.time_field].values) < 1: + raise ProviderNoDataError() + + try: + height = data.dims[self.y_field] + except KeyError: + height = 1 + try: + width = data.dims[self.x_field] + except KeyError: + width = 1 + + bbox = wkt.bounds + out_meta = { + 'bbox': [bbox[0], bbox[1], bbox[2], bbox[3]], + "time": [ + _to_datetime_string(data.coords[self.time_field].values[0]), + _to_datetime_string(data.coords[self.time_field].values[-1]) + ], + "driver": "xarray", + "height": height, + "width": width, + "time_steps": data.dims[self.time_field], + "variables": {var_name: var.attrs + for var_name, var in data.variables.items()} + } + + return self.gen_covjson(out_meta, data, self.fields) diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index ac914a4..805c309 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -467,6 +467,46 @@ async def get_process_job_result_resource(request: Request, process_id=None, return response +@app.route('/collections/{collection_id}/position') +@app.route('/collections/{collection_id}/area') +@app.route('/collections/{collection_id}/cube') +@app.route('/collections/{collection_id}/trajectory') +@app.route('/collections/{collection_id}/corridor') +@app.route('/collections/{collection_id}/instances/{instance_id}/position') +@app.route('/collections/{collection_id}/instances/{instance_id}/area') +@app.route('/collections/{collection_id}/instances/{instance_id}/cube') +@app.route('/collections/{collection_id}/instances/{instance_id}/trajectory') +@app.route('/collections/{collection_id}/instances/{instance_id}/corridor') +async def get_collection_edr_query(request: Request, collection_id=None, instance_id=None): # noqa + """ + OGC EDR API endpoints + + :param collection_id: collection identifier + :param instance_id: instance identifier + + :returns: HTTP response + """ + + if 'collection_id' in request.path_params: + collection_id = request.path_params['collection_id'] + + if 'instance_id' in request.path_params: + instance_id = request.path_params['instance_id'] + + query_type = request.path.split('/')[-1] + + headers, status_code, content = api_.get_collection_edr_query( + request.headers, request.query_params, collection_id, instance_id, + query_type) + + response = Response(content=content, status_code=status_code) + + if headers: + response.headers.update(headers) + + return response + + @app.route('/stac') async def stac_catalog_root(request: Request): """ diff --git a/pygeoapi/templates/collections/edr/query.html b/pygeoapi/templates/collections/edr/query.html new file mode 100644 index 0000000..085b12d --- /dev/null +++ b/pygeoapi/templates/collections/edr/query.html @@ -0,0 +1,80 @@ +{% extends "_base.html" %} +{% block title %}{{ super() }} {{ data['title'] }} {% endblock %} +{% block crumbs %}{{ super() }} +/ Collections +{% for link in data['links'] %} + {% if link.rel == 'collection' %} / + {{ link['title'] | truncate( 25 ) }} + {% set col_title = link['title'] %} + {% endif %} +{% endfor %} +/ Items +{% endblock %} +{% block extrahead %} + + + + + + +{% endblock %} + +{% block body %} +
+
+
+{% endblock %} + +{% block extrafoot %} +{% if data %} + + + +{% endif %} +{% endblock %} diff --git a/requirements.txt b/requirements.txt index 235f2b8..ada3276 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ python-dateutil pytz PyYAML rasterio +shapely unicodecsv diff --git a/tests/pygeoapi-test-config.yml b/tests/pygeoapi-test-config.yml index 136dd52..0159ea6 100644 --- a/tests/pygeoapi-test-config.yml +++ b/tests/pygeoapi-test-config.yml @@ -225,6 +225,32 @@ resources: name: GRIB mimetype: application/x-grib2 + icoads-sst: + type: collection + title: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + description: International Comprehensive Ocean-Atmosphere Data Set (ICOADS) + keywords: + - icoads + - sst + - air temperature + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + links: + - type: text/html + rel: canonical + title: information + href: https://psl.noaa.gov/data/gridded/data.coads.1deg.html + hreflang: en-US + providers: + - type: edr + name: xarray-edr + data: tests/data/coads_sst.nc + format: + name: NetCDF + mimetype: application/x-netcdf + hello-world: type: process processor: diff --git a/tests/test_api.py b/tests/test_api.py index 54ef9e6..e27d55d 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -177,7 +177,7 @@ def test_conformance(config, api_): assert isinstance(root, dict) assert 'conformsTo' in root - assert len(root['conformsTo']) == 16 + assert len(root['conformsTo']) == 17 rsp_headers, code, response = api_.conformance(req_headers, {'f': 'foo'}) assert code == 400 @@ -201,7 +201,7 @@ def test_describe_collections(config, api_): collections = json.loads(response) assert len(collections) == 2 - assert len(collections['collections']) == 4 + assert len(collections['collections']) == 5 assert len(collections['links']) == 3 rsp_headers, code, response = api_.describe_collections( @@ -1116,6 +1116,92 @@ def test_delete_process_job(api_): assert code == 404 +def test_get_collection_edr_query(config, api_): + # no coords parameter + req_headers = make_req_headers() + rsp_headers, code, response = api_.get_collection_edr_query( + req_headers, {}, 'icoads-sst', instance=None, query_type='position') + assert code == 400 + + # bad query type + req_headers = make_req_headers() + rsp_headers, code, response = api_.get_collection_edr_query( + req_headers, {'coords': 'POINT(11 11)'}, 'icoads-sst', instance=None, + query_type='corridor') + assert code == 400 + + # bad coords parameter + req_headers = make_req_headers() + rsp_headers, code, response = api_.get_collection_edr_query( + req_headers, {'coords': 'gah'}, 'icoads-sst', instance=None, + query_type='position') + assert code == 400 + + # bad parameter-name parameter + req_headers = make_req_headers() + rsp_headers, code, response = api_.get_collection_edr_query( + req_headers, {'coords': 'POINT(11 11)', 'parameter-name': 'bad'}, + 'icoads-sst', instance=None, query_type='position') + assert code == 400 + + # all parameters + req_headers = make_req_headers() + rsp_headers, code, response = api_.get_collection_edr_query( + req_headers, {'coords': 'POINT(11 11)'}, 'icoads-sst', instance=None, + query_type='position') + assert code == 200 + + data = json.loads(response) + + axes = list(data['domain']['axes'].keys()) + axes.sort() + assert len(axes) == 3 + assert axes == ['TIME', 'x', 'y'] + + assert data['domain']['axes']['x']['start'] == 11.0 + assert data['domain']['axes']['x']['stop'] == 11.0 + assert data['domain']['axes']['y']['start'] == 11.0 + assert data['domain']['axes']['y']['stop'] == 11.0 + + parameters = list(data['parameters'].keys()) + parameters.sort() + assert len(parameters) == 4 + assert parameters == ['AIRT', 'SST', 'UWND', 'VWND'] + + # single parameter + req_headers = make_req_headers() + rsp_headers, code, response = api_.get_collection_edr_query( + req_headers, {'coords': 'POINT(11 11)', 'parameter-name': 'SST'}, + 'icoads-sst', instance=None, query_type='position') + assert code == 200 + + data = json.loads(response) + + assert len(data['parameters'].keys()) == 1 + assert list(data['parameters'].keys())[0] == 'SST' + + # some data + req_headers = make_req_headers() + rsp_headers, code, response = api_.get_collection_edr_query( + req_headers, {'coords': 'POINT(11 11)', 'datetime': '2000-01-16'}, + 'icoads-sst', instance=None, query_type='position') + assert code == 200 + + # no data + req_headers = make_req_headers() + rsp_headers, code, response = api_.get_collection_edr_query( + req_headers, {'coords': 'POINT(11 11)', 'datetime': '2000-01-17'}, + 'icoads-sst', instance=None, query_type='position') + assert code == 204 + + # no data +# req_headers = make_req_headers() +# rsp_headers, code, response = api_.get_collection_edr_query( +# req_headers, {'coords': 'POINT(11 11)', 'datetime': '2000-01-15'}, +# 'icoads-sst', instance=None, query_type='position') +# assert code == 204 + + def test_validate_bbox(): assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] assert validate_bbox('-142,42,-52,84') == [-142, 42, -52, 84] diff --git a/tests/test_util.py b/tests/test_util.py index 5498ca6..bad9339 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -124,7 +124,7 @@ def test_filter_dict_by_key_value(): collections = util.filter_dict_by_key_value(d['resources'], 'type', 'collection') - assert len(collections) == 4 + assert len(collections) == 5 notfound = util.filter_dict_by_key_value(d['resources'], 'type', 'foo')