diff --git a/docs/source/conf.py b/docs/source/conf.py index 2809373..d5f4139 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -116,7 +116,7 @@ release = version # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index a577ee0..ba35d38 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -32,7 +32,8 @@ Reference ^^^^^^^^^^ The ``server`` section provides directives on binding and high level tuning. -Please find more information related to API design rules (the property at the bottom of the example below) :ref:`further down`. + +For more information related to API design rules (the ``api_rules`` property in the example below) see :ref:`API Design Rules`. .. code-block:: yaml @@ -312,6 +313,8 @@ Examples: curl https://example.org/collections/foo # user can access resource normally +.. _API Design Rules: + API Design Rules ---------------- diff --git a/docs/source/data-publishing/ogcapi-features.rst b/docs/source/data-publishing/ogcapi-features.rst index 9db0e2a..efb5bbe 100644 --- a/docs/source/data-publishing/ogcapi-features.rst +++ b/docs/source/data-publishing/ogcapi-features.rst @@ -21,6 +21,7 @@ parameters. `CSV`_,✅/✅,results/hits,❌,❌,❌,✅,❌,❌,✅ `Elasticsearch`_,✅/✅,results/hits,✅,✅,✅,✅,✅,✅,✅ + `ERDDAP Tabledap Service`_,❌/❌,results/hits,✅,✅,❌,❌,❌,❌ `ESRI Feature Service`_,✅/✅,results/hits,✅,✅,✅,✅,❌,❌,✅ `GeoJSON`_,✅/✅,results/hits,❌,❌,❌,✅,❌,❌,✅ `MongoDB`_,✅/❌,results,✅,✅,✅,✅,❌,❌,✅ @@ -397,14 +398,41 @@ relies on `sodapy `. .. code-block:: yaml providers: - - type: feature - name: Socrata - data: https://soda.demo.socrata.com/ - resource_id: emdb-u46w - id_field: earthquake_id - geom_field: location - time_field: datetime # Optional time_field for datetime queries - token: my_token # Optional app token + - type: feature + name: Socrata + data: https://soda.demo.socrata.com/ + resource_id: emdb-u46w + id_field: earthquake_id + geom_field: location + time_field: datetime # Optional time_field for datetime queries + token: my_token # Optional app token + + +.. _ERDDAP Tabledap Service: + +ERDDAP Tabledap Service +^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + Requires Python package `requests`_ + +To publish from an ERDDAP `Tabledap`_ service, the following are required in your index: + +.. code-block:: yaml + + providers: + - type: feature + name: ERDDAPTabledap + data: http://osmc.noaa.gov/erddap/tabledap/OSMC_Points + id_field: PLATFORM_CODE + time_field: time + options: + filters: "¶meter=\"SLP\"&platform!=\"C-MAN%20WEATHER%20STATIONS\"&platform!=\"TIDE GAUGE STATIONS (GENERIC)\"" + max_age_hours: 12 + + +.. note:: + If the ``datetime`` parameter is passed by the client, this overrides the ``options.max_age_hours`` setting. Controlling the order of properties ----------------------------------- @@ -471,3 +499,5 @@ Data access examples .. _`Google Cloud SQL`: https://cloud.google.com/sql .. _`OGC API - Features`: https://www.ogc.org/standards/ogcapi-features +.. _`Tabledap`: https://coastwatch.pfeg.noaa.gov/erddap/tabledap/documentation.html +.. _`requests`: https://requests.readthedocs.io diff --git a/pygeoapi/plugin.py b/pygeoapi/plugin.py index a2d63c2..bb0383e 100644 --- a/pygeoapi/plugin.py +++ b/pygeoapi/plugin.py @@ -42,6 +42,7 @@ PLUGINS = { 'CSV': 'pygeoapi.provider.csv_.CSVProvider', 'Elasticsearch': 'pygeoapi.provider.elasticsearch_.ElasticsearchProvider', # noqa 'ElasticsearchCatalogue': 'pygeoapi.provider.elasticsearch_.ElasticsearchCatalogueProvider', # noqa + 'ERDDAPTabledap': 'pygeoapi.provider.erddap.TabledapProvider', 'ESRI': 'pygeoapi.provider.esri.ESRIServiceProvider', 'FileSystem': 'pygeoapi.provider.filesystem.FileSystemProvider', 'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider', diff --git a/pygeoapi/provider/erddap.py b/pygeoapi/provider/erddap.py new file mode 100644 index 0000000..bcc462e --- /dev/null +++ b/pygeoapi/provider/erddap.py @@ -0,0 +1,194 @@ +# ================================================================= +# +# Authors: David Berry +# Tom Kralidis +# +# Copyright (c) 2023 David Inglis Berry +# Copyright (c) 2023 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +# feature provider for ERDDAP integrations +# +# Tabledap sample configuration +# ----------------------------- +# +# providers: +# - type: feature +# name: pygeoapi.provider.erddap.TabledapProvider +# data: http://osmc.noaa.gov/erddap/tabledap/OSMC_Points +# id_field: id +# options: +# filters: "¶meter=\"SLP\"&platform!=\"C-MAN%20WEATHER%20STATIONS\"&platform!=\"TIDE GAUGE STATIONS (GENERIC)\"" # noqa +# max_age_hours: 12 + + +from datetime import datetime, timedelta, timezone +import logging + +import requests + +from pygeoapi.provider.base import ( + BaseProvider, ProviderNotFoundError, ProviderQueryError) + +LOGGER = logging.getLogger(__name__) + + +class TabledapProvider(BaseProvider): + def __init__(self, provider_def): + super().__init__(provider_def) + + LOGGER.debug('Setting provider query filters') + self.filters = self.options.get('filters') + self.fields = self.get_fields() + + def get_fields(self): + LOGGER.debug('Fetching one feature for field definitions') + properties = self.query(limit=1)['features'][0]['properties'] + + for key, value in properties.items(): + LOGGER.debug(f'Field: {key}={value}') + properties[key] = {'type': type(value).__name__} + + return properties + + def query(self, startindex=0, limit=10, resulttype='results', + bbox=[], datetime_=None, properties=[], sortby=[], + select_properties=[], skip_geometry=False, q=None, + filterq=None, **kwargs): + + query_params = [] + + max_age_hours = self.options.get('max_age_hours') + url = f'{self.data}.geoJson' + + if self.filters is not None: + LOGGER.debug(f'Setting filters ({self.filters})') + query_params.append(self.filters) + + if max_age_hours is not None: + LOGGER.debug(f'Setting default time filter {max_age_hours} hours') + currenttime = datetime.now(timezone.utc) + mintime = currenttime - timedelta(hours=max_age_hours) + mintime = mintime.strftime('%Y-%m-%dT%H:%M:%SZ') + query_params.append(f'time>={mintime}') + + elif datetime_ is not None: + LOGGER.debug('Setting datetime filters') + + LOGGER.debug('Setting datetime filters') + if '/' in datetime_: # envelope + LOGGER.debug('detected time range') + time_begin, time_end = datetime_.split('/') + + if time_begin != '..': + LOGGER.debug('Setting time_begin') + query_params.append(f'time>={time_begin}') + if time_end != '..': + LOGGER.debug('Setting time_end') + query_params.append(f'time<={time_end}') + else: + query_params.append(f'time={datetime_}') + + if bbox: + LOGGER.debug('Setting bbox') + query_params.extend([ + f'latitude>={bbox[1]}', + f'latitude<={bbox[3]}', + f'longitude>={bbox[0]}', + f'longitude<={bbox[2]}' + ]) + + url = f'{url}?{"&".join(query_params)}' + + LOGGER.debug(f'Fetching data from {url}') + response = requests.get(url) + LOGGER.debug(f'Response: {response}') + data = response.json() + LOGGER.debug(f'Data: {data}') + + matched = len(data['features']) + returned = limit + + data = data['features'][startindex:limit] + + # add id to each feature as this is required by pygeoapi + for idx in range(len(data)): + # ID used to extract individual features + try: + id_ = data[idx]['properties'][self.id_field] + except KeyError: + # ERDDAP changes case of parameters depending on result + id_ = data[idx]['properties'][self.id_field] + except Exception as err: + msg = 'Cannot determine station identifier' + LOGGER.error(msg, err) + raise ProviderQueryError(msg) + + obs_time = data[idx]['properties']['time'] + obs_id = f'{id_}.{obs_time}' + data[idx]['id'] = obs_id + + return { + 'type': 'FeatureCollection', + 'features': data, + 'numberMatched': matched, + 'numberReturned': returned + } + + def get(self, identifier, **kwargs): + + query_params = [] + + url = f'{self.data}.geoJson' + + if self.filters is not None: + LOGGER.debug(f'Setting filters ({self.filters})') + query_params.append(self.filters) + + id_, obs_time = identifier.split('.') + + query_params.extend([ + f'time={obs_time}', + f'{self.id_field}=%22{id_}%22' + ]) + + url = f'{url}?{"&".join(query_params)}' + LOGGER.debug(f'Fetching data from {url}') + + response = requests.get(url) + LOGGER.debug(f'Response: {response}') + data = response.json() + LOGGER.debug(f'Data: {data}') + + if len(data['features']) < 1: + msg = 'No features found' + LOGGER.error(msg) + raise ProviderNotFoundError(msg) + + LOGGER.debug('Truncating to first feature') + data = data['features'][0] + data['id'] = identifier + + return data