From 35bdcb6f024f261b1c784dcd848f31fc71e4835c Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Fri, 5 Apr 2024 12:06:25 +0200 Subject: [PATCH] Proposal to split api.py into different files (#1405) * Move api to subdirectory * Move processes api to own file * Adapt processes view methods * Move openapi definition to processes api * Use processes api in flask * Linter * Fix import issues * Allow calling refactored views from starlette * Allow calling refactored views from django * Linter * Move edr api to own file * Adapt edr api to new style * Fix typo in django views * Move maps api to own file * Adapt maps api to new style * Move edr openapi to edr api file * Move maps openapi to maps api file * Move stac views to own file * Refactor stac views to new file * Move stac openapi to stac api file * Move tiles api to own file * Adapt tiles api to new style * Also move tilematrixset to tiles api * Adapt tilesetmatrix views to new style NOTE: I had to remove one tilematrixsets test because it tested that an invalid format would produce an error. This now happens by default for all views, but the actual code is outside of the endpoint function. * update features, records, coverages * update release version * switch back to dev * backport of #1313 * backport of #1313 fix * backport of #1585 * Flask: sanitize OGC schema pathing (#1593) * update release version * switch back to dev * backport of #1596 * Port test_gzip_csv test Note that apply_gzip is now called by the web framework adapters, so to test it in general, we have to call it in the test manually * Add empty conformance class list to stac api * Fix queryables call in starlette * fix ref * Unify request validity checking The default case is handled by the web framework adapters. If custom format handling is required, the check in the adapter must be skipped. * Fix imports in django views * backport #1598 * Remove test about format handling in endpoint This is now handled outside of the endpoint function * add docstring to base process manager (#1603) * backport of #1601 * Port api ogr tests to new style * Move processes tests to own file * Run api tests from new dir in CI * Move edr tests to own file * Move maps tests to own file * Move tiles tests to own file * Actually hide hidden layers in openapi * 1600 allow providing default value in config (#1604) * move coverages tests to own file * move itemtypes to own file, move core into init test * fix OpenAPI output * update tests * add missing descriptions to OpenAPI admin responses * update tests * fix tests autodiscovery * remove unused logging in tests * address PR comments * test with xarray 2024.2.0 * remove unneeded file * safeguard xarray error * unpin xarray * fix OpenAPI generation * fix schema endpoint in Flask and Starlette * Safely serialize configuration JSON (#1605) * Safely serialize configuration JSON Co-Authored-By: Tom Kralidis * Revert "Safely serialize configuration JSON" This reverts commit 36feb067ee6f87e61955852dc48994f075806370. * Add test for datetime with Admin API * Safely serialize configuration JSON --------- Co-authored-by: Tom Kralidis * backport #1611 * Also fix schema endpoint for django Fix is analogous to e72d4ba3a5ba3b8621ca839e7814429beeeb8f01 * address additional PR comments --------- Co-authored-by: Tom Kralidis Co-authored-by: Angelos Tzotsos Co-authored-by: Ricardo Garcia Silva Co-authored-by: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> --- .github/workflows/main.yml | 2 +- docs/source/development.rst | 2 +- pygeoapi/api.py | 4319 ----------------- pygeoapi/api/__init__.py | 1687 +++++++ pygeoapi/api/coverages.py | 253 + pygeoapi/api/environmental_data_retrieval.py | 332 ++ pygeoapi/api/itemtypes.py | 1610 ++++++ pygeoapi/api/maps.py | 340 ++ pygeoapi/api/processes.py | 740 +++ pygeoapi/api/stac.py | 256 + pygeoapi/api/tiles.py | 534 ++ pygeoapi/django_/views.py | 144 +- pygeoapi/flask_app.py | 170 +- pygeoapi/openapi.py | 1158 +---- pygeoapi/process/manager/base.py | 6 +- pygeoapi/process/manager/dummy.py | 2 +- pygeoapi/provider/xarray_.py | 11 +- pygeoapi/starlette_app.py | 155 +- pygeoapi/util.py | 4 +- tests/api/test_api.py | 869 ++++ tests/api/test_coverages.py | 177 + .../api/test_environmental_data_retrieval.py | 226 + tests/api/test_itemtypes.py | 659 +++ tests/api/test_maps.py | 52 + tests/api/test_processes.py | 428 ++ tests/api/test_tiles.py | 110 + tests/conftest.py | 52 + tests/test_api.py | 2262 --------- tests/test_api_ogr_provider.py | 50 +- tests/test_postgresql_provider.py | 99 +- tests/util.py | 21 + 31 files changed, 8918 insertions(+), 7812 deletions(-) delete mode 100644 pygeoapi/api.py create mode 100644 pygeoapi/api/__init__.py create mode 100644 pygeoapi/api/coverages.py create mode 100644 pygeoapi/api/environmental_data_retrieval.py create mode 100644 pygeoapi/api/itemtypes.py create mode 100644 pygeoapi/api/maps.py create mode 100644 pygeoapi/api/processes.py create mode 100644 pygeoapi/api/stac.py create mode 100644 pygeoapi/api/tiles.py create mode 100644 tests/api/test_api.py create mode 100644 tests/api/test_coverages.py create mode 100644 tests/api/test_environmental_data_retrieval.py create mode 100644 tests/api/test_itemtypes.py create mode 100644 tests/api/test_maps.py create mode 100644 tests/api/test_processes.py create mode 100644 tests/api/test_tiles.py create mode 100644 tests/conftest.py delete mode 100644 tests/test_api.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index de37862..3368f98 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,7 +109,7 @@ jobs: env: POSTGRESQL_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }} run: | - pytest tests/test_api.py + pytest tests/api pytest tests/test_api_ogr_provider.py pytest tests/test_config.py pytest tests/test_csv__formatter.py diff --git a/docs/source/development.rst b/docs/source/development.rst index 561f083..59941d7 100644 --- a/docs/source/development.rst +++ b/docs/source/development.rst @@ -20,7 +20,7 @@ Tests can be run locally as part of development workflow. They are also run on `GitHub Actions setup`_ against all commits and pull requests to the code repository. To run all tests, simply run ``pytest`` in the repository. To run a specific test file, -run ``pytest tests/test_api.py``, for example. +run ``pytest tests/api/test_itemtypes.py``, for example. CQL extension lifecycle diff --git a/pygeoapi/api.py b/pygeoapi/api.py deleted file mode 100644 index 87d672c..0000000 --- a/pygeoapi/api.py +++ /dev/null @@ -1,4319 +0,0 @@ -# ================================================================= -# -# Authors: Tom Kralidis -# Francesco Bartoli -# Sander Schaminee -# John A Stevenson -# Colin Blackburn -# Ricardo Garcia Silva -# -# Copyright (c) 2024 Tom Kralidis -# Copyright (c) 2022 Francesco Bartoli -# Copyright (c) 2022 John A Stevenson and Colin Blackburn -# Copyright (c) 2023 Ricardo Garcia Silva -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= -""" Root level code of pygeoapi, parsing content provided by web framework. -Returns content from plugins and sets responses. -""" - -import asyncio -from collections import OrderedDict -from copy import deepcopy -from datetime import datetime, timezone -from functools import partial -from gzip import compress -from http import HTTPStatus -import json -import logging -import re -from typing import Any, Tuple, Union, Optional -import urllib.parse - -from dateutil.parser import parse as dateparse -from pygeofilter.parsers.ecql import parse as parse_ecql_text -from pygeofilter.parsers.cql_json import parse as parse_cql_json -from pyproj.exceptions import CRSError -import pytz -from shapely.errors import WKTReadingError -from shapely.wkt import loads as shapely_loads - -from pygeoapi import __version__, l10n -from pygeoapi.formatter.base import FormatterSerializationError -from pygeoapi.linked_data import (geojson2jsonld, jsonldify, - jsonldify_collection) -from pygeoapi.log import setup_logger -from pygeoapi.process.base import ( - JobNotFoundError, - JobResultNotFoundError, - ProcessorExecuteError, -) -from pygeoapi.process.manager.base import get_manager -from pygeoapi.plugin import load_plugin, PLUGINS -from pygeoapi.provider.base import ( - ProviderGenericError, ProviderConnectionError, ProviderNotFoundError, - ProviderTypeError) -from pygeoapi.models.provider.base import (TilesMetadataFormat, - TileMatrixSetEnum) - -from pygeoapi.models.cql import CQLModel -from pygeoapi.util import (dategetter, RequestedProcessExecutionMode, - DATETIME_FORMAT, UrlPrefetcher, - filter_dict_by_key_value, get_provider_by_type, - get_provider_default, get_typed_value, JobStatus, - json_serial, render_j2_template, str2bool, - TEMPLATES, to_json, get_api_rules, get_base_url, - get_crs_from_uri, get_supported_crs_list, - modify_pygeofilter, CrsTransformSpec, - transform_bbox, Subscriber) - -LOGGER = logging.getLogger(__name__) - -#: Return headers for requests (e.g:X-Powered-By) -HEADERS = { - 'Content-Type': 'application/json', - 'X-Powered-By': f'pygeoapi {__version__}' -} - -CHARSET = ['utf-8'] -F_JSON = 'json' -F_HTML = 'html' -F_JSONLD = 'jsonld' -F_GZIP = 'gzip' -F_PNG = 'png' -F_JPEG = 'jpeg' -F_MVT = 'mvt' -F_NETCDF = 'NetCDF' - -#: Formats allowed for ?f= requests (order matters for complex MIME types) -FORMAT_TYPES = OrderedDict(( - (F_HTML, 'text/html'), - (F_JSONLD, 'application/ld+json'), - (F_JSON, 'application/json'), - (F_PNG, 'image/png'), - (F_JPEG, 'image/jpeg'), - (F_MVT, 'application/vnd.mapbox-vector-tile'), - (F_NETCDF, 'application/x-netcdf'), -)) - -#: Locale used for system responses (e.g. exceptions) -SYSTEM_LOCALE = l10n.Locale('en', 'US') - -CONFORMANCE = { - 'common': [ - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/landing-page', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/oas30' - ], - 'feature': [ - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/req/oas30', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson', - 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs', - 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables', - 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables-query-parameters', # noqa - 'http://www.opengis.net/spec/ogcapi-features-4/1.0/conf/create-replace-delete', # noqa - 'http://www.opengis.net/spec/ogcapi-features-5/1.0/conf/schemas', - 'http://www.opengis.net/spec/ogcapi-features-5/1.0/req/core-roles-features' # noqa - ], - 'coverage': [ - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/oas30', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/html', - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/geodata-coverage', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-subset', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-rangesubset', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-bbox', # noqa - 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-datetime' # noqa - ], - 'map': [ - 'http://www.opengis.net/spec/ogcapi-maps-1/1.0/conf/core' - ], - 'tile': [ - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/mvt', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tileset', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tilesets-list', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/oas30', - 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/geodata-tilesets' - ], - 'record': [ - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/sorting', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/opensearch', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/html' - ], - 'process': [ - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/ogc-process-description', # noqa - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core', - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json', - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30' - 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback', - ], - 'edr': [ - 'http://www.opengis.net/spec/ogcapi-edr-1/1.0/conf/core' - ] -} - -OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' - -DEFAULT_CRS_LIST = [ - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', -] - -DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' -DEFAULT_STORAGE_CRS = DEFAULT_CRS - - -def pre_process(func): - """ - Decorator that transforms an incoming Request instance specific to the - web framework (i.e. Flask, Starlette or Django) into a generic - :class:`APIRequest` instance. - - :param func: decorated function - - :returns: `func` - """ - - def inner(*args): - cls, req_in = args[:2] - req_out = APIRequest.with_data(req_in, getattr(cls, 'locales', set())) - if len(args) > 2: - return func(cls, req_out, *args[2:]) - else: - return func(cls, req_out) - - return inner - - -def gzip(func): - """ - Decorator that compresses the content of an outgoing API result - instance if the Content-Encoding response header was set to gzip. - - :param func: decorated function - - :returns: `func` - """ - - def inner(*args, **kwargs): - headers, status, content = func(*args, **kwargs) - charset = CHARSET[0] - if F_GZIP in headers.get('Content-Encoding', []): - try: - if isinstance(content, bytes): - # bytes means Content-Type needs to be set upstream - content = compress(content) - else: - headers['Content-Type'] = \ - f"{headers['Content-Type']}; charset={charset}" - content = compress(content.encode(charset)) - except TypeError as err: - headers.pop('Content-Encoding') - LOGGER.error(f'Error in compression: {err}') - - return headers, status, content - - return inner - - -class APIRequest: - """ - Transforms an incoming server-specific Request into an object - with some generic helper methods and properties. - - .. note:: Typically, this instance is created automatically by the - :func:`pre_process` decorator. **Every** API method that has - been routed to a REST endpoint should be decorated by the - :func:`pre_process` function. - Therefore, **all** routed API methods should at least have 1 - argument that holds the (transformed) request. - - The following example API method will: - - - transform the incoming Flask/Starlette/Django `Request` into an - `APIRequest`using the :func:`pre_process` decorator; - - call :meth:`is_valid` to check if the incoming request was valid, i.e. - that the user requested a valid output format or no format at all - (which means the default format); - - call :meth:`API.get_format_exception` if the requested format was - invalid; - - create a `dict` with the appropriate `Content-Type` header for the - requested format and a `Content-Language` header if any specific language - was requested. - - .. code-block:: python - - @pre_process - def example_method(self, request: Union[APIRequest, Any], custom_arg): - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers() - - # generate response_body here - - return headers, HTTPStatus.OK, response_body - - - The following example API method is similar as the one above, but will also - allow the user to request a non-standard format (e.g. ``f=xml``). - If `xml` was requested, we set the `Content-Type` ourselves. For the - standard formats, the `APIRequest` object sets the `Content-Type`. - - .. code-block:: python - - @pre_process - def example_method(self, request: Union[APIRequest, Any], custom_arg): - if not request.is_valid(['xml']): - return self.get_format_exception(request) - - content_type = 'application/xml' if request.format == 'xml' else None - headers = request.get_response_headers(content_type) - - # generate response_body here - - return headers, HTTPStatus.OK, response_body - - Note that you don't *have* to call :meth:`is_valid`, but that you can also - perform a custom check on the requested output format by looking at the - :attr:`format` property. - Other query parameters are available through the :attr:`params` property as - a `dict`. The request body is available through the :attr:`data` property. - - .. note:: If the request data (body) is important, **always** create a - new `APIRequest` instance using the :meth:`with_data` factory - method. - The :func:`pre_process` decorator will use this automatically. - - :param request: The web platform specific Request instance. - :param supported_locales: List or set of supported Locale instances. - """ - def __init__(self, request, supported_locales): - # Set default request data - self._data = b'' - - # Copy request query parameters - self._args = self._get_params(request) - - # Get path info - if hasattr(request, 'scope'): - self._path_info = request.scope['path'].strip('/') - elif hasattr(request.headers, 'environ'): - self._path_info = request.headers.environ['PATH_INFO'].strip('/') - elif hasattr(request, 'path_info'): - self._path_info = request.path_info - - # Extract locale from params or headers - self._raw_locale, self._locale = self._get_locale(request.headers, - supported_locales) - - # Determine format - self._format = self._get_format(request.headers) - - # Get received headers - self._headers = self.get_request_headers(request.headers) - - @classmethod - def with_data(cls, request, supported_locales) -> 'APIRequest': - """ - Factory class method to create an `APIRequest` instance with data. - - If the request body is required, an `APIRequest` should always be - instantiated using this class method. The reason for this is, that the - Starlette request body needs to be awaited (async), which cannot be - achieved in the :meth:`__init__` method of the `APIRequest`. - However, `APIRequest` can still be initialized using :meth:`__init__`, - but then the :attr:`data` property value will always be empty. - - :param request: The web platform specific Request instance. - :param supported_locales: List or set of supported Locale instances. - :returns: An `APIRequest` instance with data. - """ - - api_req = cls(request, supported_locales) - if hasattr(request, 'data'): - # Set data from Flask request - api_req._data = request.data - elif hasattr(request, 'body'): - if 'django' in str(request.__class__): - # Set data from Django request - api_req._data = request.body - else: - # Set data from Starlette request after async - # coroutine completion - # TODO: - # this now blocks, but once Flask v2 with async support - # has been implemented, with_data() can become async too - loop = asyncio.get_event_loop() - api_req._data = asyncio.run_coroutine_threadsafe( - request.body(), loop).result(1) - return api_req - - @staticmethod - def _get_params(request): - """ - Extracts the query parameters from the `Request` object. - - :param request: A Flask or Starlette Request instance - :returns: `ImmutableMultiDict` or empty `dict` - """ - - if hasattr(request, 'args'): - # Return ImmutableMultiDict from Flask request - return request.args - elif hasattr(request, 'query_params'): - # Return ImmutableMultiDict from Starlette request - return request.query_params - elif hasattr(request, 'GET'): - # Return QueryDict from Django GET request - return request.GET - elif hasattr(request, 'POST'): - # Return QueryDict from Django GET request - return request.POST - LOGGER.debug('No query parameters found') - return {} - - def _get_locale(self, headers, supported_locales): - """ - Detects locale from "lang=" param or `Accept-Language` - header. Returns a tuple of (raw, locale) if found in params or headers. - Returns a tuple of (raw default, default locale) if not found. - - :param headers: A dict with Request headers - :param supported_locales: List or set of supported Locale instances - :returns: A tuple of (str, Locale) - """ - - raw = None - try: - default_locale = l10n.str2locale(supported_locales[0]) - except (TypeError, IndexError, l10n.LocaleError) as err: - # This should normally not happen, since the API class already - # loads the supported languages from the config, which raises - # a LocaleError if any of these languages are invalid. - LOGGER.error(err) - raise ValueError(f"{self.__class__.__name__} must be initialized" - f"with a list of valid supported locales") - - for func, mapping in ((l10n.locale_from_params, self._args), - (l10n.locale_from_headers, headers)): - loc_str = func(mapping) - if loc_str: - if not raw: - # This is the first-found locale string: set as raw - raw = loc_str - # Check if locale string is a good match for the UI - loc = l10n.best_match(loc_str, supported_locales) - is_override = func is l10n.locale_from_params - if loc != default_locale or is_override: - return raw, loc - - return raw, default_locale - - def _get_format(self, headers) -> Union[str, None]: - """ - Get `Request` format type from query parameters or headers. - - :param headers: Dict of Request headers - :returns: format value or None if not found/specified - """ - - # Optional f=html or f=json query param - # Overrides Accept header and might differ from FORMAT_TYPES - format_ = (self._args.get('f') or '').strip() - if format_: - return format_ - - # Format not specified: get from Accept headers (MIME types) - # e.g. format_ = 'text/html' - h = headers.get('accept', headers.get('Accept', '')).strip() # noqa - (fmts, mimes) = zip(*FORMAT_TYPES.items()) - # basic support for complex types (i.e. with "q=0.x") - for type_ in (t.split(';')[0].strip() for t in h.split(',') if t): - if type_ in mimes: - idx_ = mimes.index(type_) - format_ = fmts[idx_] - break - - return format_ or None - - @property - def data(self) -> bytes: - """Returns the additional data send with the Request (bytes)""" - return self._data - - @property - def params(self) -> dict: - """Returns the Request query parameters dict""" - return self._args - - @property - def path_info(self) -> str: - """Returns the web server request path info part""" - return self._path_info - - @property - def locale(self) -> l10n.Locale: - """ - Returns the user-defined locale from the request object. - If no locale has been defined or if it is invalid, - the default server locale is returned. - - .. note:: The locale here determines the language in which pygeoapi - should return its responses. This may not be the language - that the user requested. It may also not be the language - that is supported by a collection provider, for example. - For this reason, you should pass the `raw_locale` property - to the :func:`l10n.get_plugin_locale` function, so that - the best match for the provider can be determined. - - :returns: babel.core.Locale - """ - - return self._locale - - @property - def raw_locale(self) -> Union[str, None]: - """ - Returns the raw locale string from the `Request` object. - If no "lang" query parameter or `Accept-Language` header was found, - `None` is returned. - Pass this value to the :func:`l10n.get_plugin_locale` function to let - the provider determine a best match for the locale, which may be - different from the locale used by pygeoapi's UI. - - :returns: a locale string or None - """ - - return self._raw_locale - - @property - def format(self) -> Union[str, None]: - """ - Returns the content type format from the - request query parameters or headers. - - :returns: Format name or None - """ - - return self._format - - @property - def headers(self) -> dict: - """ - Returns the dictionary of the headers from - the request. - - :returns: Request headers dictionary - """ - - return self._headers - - def get_linkrel(self, format_: str) -> str: - """ - Returns the hyperlink relationship (rel) attribute value for - the given API format string. - - The string is compared against the request format and if it matches, - the value 'self' is returned. Otherwise, 'alternate' is returned. - However, if `format_` is 'json' and *no* request format was found, - the relationship 'self' is returned as well (JSON is the default). - - :param format_: The format to compare the request format against. - :returns: A string 'self' or 'alternate'. - """ - - fmt = format_.lower() - if fmt == self._format or (fmt == F_JSON and not self._format): - return 'self' - return 'alternate' - - def is_valid(self, additional_formats=None) -> bool: - """ - Returns True if: - - the format is not set (None) - - the requested format is supported - - the requested format exists in a list if additional formats - - .. note:: Format names are matched in a case-insensitive manner. - - :param additional_formats: Optional additional supported formats list - - :returns: bool - """ - - if not self._format: - return True - if self._format in FORMAT_TYPES.keys(): - return True - if self._format in (f.lower() for f in (additional_formats or ())): - return True - return False - - def get_response_headers(self, force_lang: l10n.Locale = None, - force_type: str = None, - force_encoding: str = None, - **custom_headers) -> dict: - """ - Prepares and returns a dictionary with Response object headers. - - This method always adds a 'Content-Language' header, where the value - is determined by the 'lang' query parameter or 'Accept-Language' - header from the request. - If no language was requested, the default pygeoapi language is used, - unless a `force_lang` override was specified (see notes below). - - A 'Content-Type' header is also always added to the response. - If the user does not specify `force_type`, the header is based on - the `format` APIRequest property. If that is invalid, the default MIME - type `application/json` is used. - - ..note:: If a `force_lang` override is applied, that language - is always set as the 'Content-Language', regardless of - a 'lang' query parameter or 'Accept-Language' header. - If an API response always needs to be in the same - language, 'force_lang' should be set to that language. - - :param force_lang: An optional Content-Language header override. - :param force_type: An optional Content-Type header override. - :param force_encoding: An optional Content-Encoding header override. - :returns: A header dict - """ - - headers = HEADERS.copy() - headers.update(**custom_headers) - l10n.set_response_language(headers, force_lang or self._locale) - if force_type: - # Set custom MIME type if specified - headers['Content-Type'] = force_type - elif self.is_valid() and self._format: - # Set MIME type for valid formats - headers['Content-Type'] = FORMAT_TYPES[self._format] - - if F_GZIP in FORMAT_TYPES: - if force_encoding: - headers['Content-Encoding'] = force_encoding - elif F_GZIP in self._headers.get('Accept-Encoding', ''): - headers['Content-Encoding'] = F_GZIP - - return headers - - def get_request_headers(self, headers) -> dict: - """ - Obtains and returns a dictionary with Request object headers. - - This method adds the headers of the original request and - makes them available to the API object. - - :returns: A header dict - """ - - headers_ = {item[0]: item[1] for item in headers.items()} - return headers_ - - -class API: - """API object""" - - def __init__(self, config, openapi): - """ - constructor - - :param config: configuration dict - :param openapi: openapi dict - - :returns: `pygeoapi.API` instance - """ - - self.config = config - self.openapi = openapi - self.api_headers = get_api_rules(self.config).response_headers - self.base_url = get_base_url(self.config) - self.prefetcher = UrlPrefetcher() - - CHARSET[0] = config['server'].get('encoding', 'utf-8') - if config['server'].get('gzip'): - FORMAT_TYPES[F_GZIP] = 'application/gzip' - FORMAT_TYPES.move_to_end(F_JSON) - - # Process language settings (first locale is default!) - self.locales = l10n.get_locales(config) - self.default_locale = self.locales[0] - - if 'templates' not in self.config['server']: - self.config['server']['templates'] = {'path': TEMPLATES} - - if 'pretty_print' not in self.config['server']: - self.config['server']['pretty_print'] = False - - self.pretty_print = self.config['server']['pretty_print'] - - setup_logger(self.config['logging']) - - # Create config clone for HTML templating with modified base URL - self.tpl_config = deepcopy(self.config) - self.tpl_config['server']['url'] = self.base_url - - self.manager = get_manager(self.config) - LOGGER.info('Process manager plugin loaded') - - @gzip - @pre_process - @jsonldify - def landing_page(self, - request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: - """ - Provide API landing page - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - fcm = { - 'links': [], - 'title': l10n.translate( - self.config['metadata']['identification']['title'], - request.locale), - 'description': - l10n.translate( - self.config['metadata']['identification']['description'], - request.locale) - } - - LOGGER.debug('Creating links') - # TODO: put title text in config or translatable files? - fcm['links'] = [{ - 'rel': request.get_linkrel(F_JSON), - 'type': FORMAT_TYPES[F_JSON], - 'title': 'This document as JSON', - 'href': f"{self.base_url}?f={F_JSON}" - }, { - 'rel': request.get_linkrel(F_JSONLD), - 'type': FORMAT_TYPES[F_JSONLD], - 'title': 'This document as RDF (JSON-LD)', - 'href': f"{self.base_url}?f={F_JSONLD}" - }, { - 'rel': request.get_linkrel(F_HTML), - 'type': FORMAT_TYPES[F_HTML], - 'title': 'This document as HTML', - 'href': f"{self.base_url}?f={F_HTML}", - 'hreflang': self.default_locale - }, { - 'rel': 'service-desc', - 'type': 'application/vnd.oai.openapi+json;version=3.0', - 'title': 'The OpenAPI definition as JSON', - 'href': f"{self.base_url}/openapi" - }, { - 'rel': 'service-doc', - 'type': FORMAT_TYPES[F_HTML], - 'title': 'The OpenAPI definition as HTML', - 'href': f"{self.base_url}/openapi?f={F_HTML}", - 'hreflang': self.default_locale - }, { - 'rel': 'conformance', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Conformance', - 'href': f"{self.base_url}/conformance" - }, { - 'rel': 'data', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Collections', - 'href': self.get_collections_url() - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/processes', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Processes', - 'href': f"{self.base_url}/processes" - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Jobs', - 'href': f"{self.base_url}/jobs" - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'The list of supported tiling schemes (as JSON)', - 'href': f"{self.base_url}/TileMatrixSets?f=json" - }, { - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', - 'type': FORMAT_TYPES[F_HTML], - 'title': 'The list of supported tiling schemes (as HTML)', - 'href': f"{self.base_url}/TileMatrixSets?f=html" - }] - - headers = request.get_response_headers(**self.api_headers) - if request.format == F_HTML: # render - - fcm['processes'] = False - fcm['stac'] = False - fcm['collection'] = False - - if filter_dict_by_key_value(self.config['resources'], - 'type', 'process'): - fcm['processes'] = True - - if filter_dict_by_key_value(self.config['resources'], - 'type', 'stac-collection'): - fcm['stac'] = True - - if filter_dict_by_key_value(self.config['resources'], - 'type', 'collection'): - fcm['collection'] = True - - content = render_j2_template(self.tpl_config, 'landing_page.html', - fcm, request.locale) - return headers, HTTPStatus.OK, content - - if request.format == F_JSONLD: - return headers, HTTPStatus.OK, to_json( - self.fcmld, self.pretty_print) - - return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) - - @gzip - @pre_process - def openapi_(self, request: Union[APIRequest, Any]) -> Tuple[ - dict, int, str]: - """ - Provide OpenAPI document - - :param request: A request object - :param openapi: dict of OpenAPI definition - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) - - if request.format == F_HTML: - template = 'openapi/swagger.html' - if request._args.get('ui') == 'redoc': - template = 'openapi/redoc.html' - - path = f'{self.base_url}/openapi' - data = { - 'openapi-document-path': path - } - content = render_j2_template(self.tpl_config, template, data, - request.locale) - return headers, HTTPStatus.OK, content - - headers['Content-Type'] = 'application/vnd.oai.openapi+json;version=3.0' # noqa - - if isinstance(self.openapi, dict): - return headers, HTTPStatus.OK, to_json(self.openapi, - self.pretty_print) - else: - return headers, HTTPStatus.OK, self.openapi - - @gzip - @pre_process - def conformance(self, - request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: - """ - Provide conformance definition - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - conformance_list = CONFORMANCE['common'] - - for key, value in self.config['resources'].items(): - if value['type'] == 'process': - conformance_list.extend(CONFORMANCE[value['type']]) - else: - for provider in value['providers']: - if provider['type'] in CONFORMANCE: - conformance_list.extend(CONFORMANCE[provider['type']]) - - conformance = { - 'conformsTo': list(set(conformance_list)) - } - - headers = request.get_response_headers(**self.api_headers) - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, 'conformance.html', - conformance, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(conformance, self.pretty_print) - - @gzip - @pre_process - def tilematrixsets(self, - request: Union[APIRequest, Any]) -> Tuple[dict, int, - str]: - """ - Provide tileMatrixSets definition - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) - - # Retrieve available TileMatrixSets - enums = [e.value for e in TileMatrixSetEnum] - - tms = {"tileMatrixSets": []} - - for e in enums: - tms['tileMatrixSets'].append({ - "title": e.title, - "id": e.tileMatrixSet, - "uri": e.tileMatrixSetURI, - "links": [ - { - "rel": "self", - "type": "text/html", - "title": f"The HTML representation of the {e.tileMatrixSet} tile matrix set", # noqa - "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=html" # noqa - }, - { - "rel": "self", - "type": "application/json", - "title": f"The JSON representation of the {e.tileMatrixSet} tile matrix set", # noqa - "href": f"{self.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=json" # noqa - } - ] - }) - - tms['links'] = [{ - "rel": "alternate", - "type": "text/html", - "title": "This document as HTML", - "href": f"{self.base_url}/tileMatrixSets?f=html" - }, { - "rel": "self", - "type": "application/json", - "title": "This document", - "href": f"{self.base_url}/tileMatrixSets?f=json" - }] - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'tilematrixsets/index.html', - tms, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) - - @gzip - @pre_process - def tilematrixset(self, - request: Union[APIRequest, Any], - tileMatrixSetId) -> Tuple[dict, - int, str]: - """ - Provide tile matrix definition - - :param request: A request object - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - headers = request.get_response_headers(**self.api_headers) - - # Retrieve relevant TileMatrixSet - enums = [e.value for e in TileMatrixSetEnum] - enum = None - - try: - for e in enums: - if tileMatrixSetId == e.tileMatrixSet: - enum = e - if not enum: - raise ValueError('could not find this tilematrixset') - except ValueError as err: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', str(err)) - - tms = { - "title": enum.tileMatrixSet, - "crs": enum.crs, - "id": enum.tileMatrixSet, - "uri": enum.tileMatrixSetURI, - "orderedAxes": enum.orderedAxes, - "wellKnownScaleSet": enum.wellKnownScaleSet, - "tileMatrices": enum.tileMatrices - } - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'tilematrixsets/tilematrixset.html', - tms, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tms, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def describe_collections(self, request: Union[APIRequest, Any], - dataset=None) -> Tuple[dict, int, str]: - """ - Provide collection metadata - - :param request: A request object - :param dataset: name of collection - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - fcm = { - 'collections': [], - 'links': [] - } - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if all([dataset is not None, dataset not in collections.keys()]): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - if dataset is not None: - collections_dict = { - k: v for k, v in collections.items() if k == dataset - } - else: - collections_dict = collections - - LOGGER.debug('Creating collections') - for k, v in collections_dict.items(): - if v.get('visibility', 'default') == 'hidden': - LOGGER.debug(f'Skipping hidden layer: {k}') - continue - collection_data = get_provider_default(v['providers']) - collection_data_type = collection_data['type'] - - collection_data_format = None - - if 'format' in collection_data: - collection_data_format = collection_data['format'] - - is_vector_tile = (collection_data_type == 'tile' and - collection_data_format['name'] not - in [F_PNG, F_JPEG]) - - collection = { - 'id': k, - 'title': l10n.translate(v['title'], request.locale), - 'description': l10n.translate(v['description'], request.locale), # noqa - 'keywords': l10n.translate(v['keywords'], request.locale), - 'links': [] - } - - bbox = v['extents']['spatial']['bbox'] - # The output should be an array of bbox, so if the user only - # provided a single bbox, wrap it in a array. - if not isinstance(bbox[0], list): - bbox = [bbox] - collection['extent'] = { - 'spatial': { - 'bbox': bbox - } - } - if 'crs' in v['extents']['spatial']: - collection['extent']['spatial']['crs'] = \ - v['extents']['spatial']['crs'] - - t_ext = v.get('extents', {}).get('temporal', {}) - if t_ext: - begins = dategetter('begin', t_ext) - ends = dategetter('end', t_ext) - collection['extent']['temporal'] = { - 'interval': [[begins, ends]] - } - if 'trs' in t_ext: - collection['extent']['temporal']['trs'] = t_ext['trs'] - - LOGGER.debug('Processing configured collection links') - for link in l10n.translate(v.get('links', []), request.locale): - lnk = { - 'type': link['type'], - 'rel': link['rel'], - 'title': l10n.translate(link['title'], request.locale), - 'href': l10n.translate(link['href'], request.locale), - } - if 'hreflang' in link: - lnk['hreflang'] = l10n.translate( - link['hreflang'], request.locale) - content_length = link.get('length', 0) - - if lnk['rel'] == 'enclosure' and content_length == 0: - # Issue HEAD request for enclosure links without length - lnk_headers = self.prefetcher.get_headers(lnk['href']) - content_length = int(lnk_headers.get('content-length', 0)) - content_type = lnk_headers.get('content-type', lnk['type']) - if content_length == 0: - # Skip this (broken) link - LOGGER.debug(f"Enclosure {lnk['href']} is invalid") - continue - if content_type != lnk['type']: - # Update content type if different from specified - lnk['type'] = content_type - LOGGER.debug( - f"Fixed media type for enclosure {lnk['href']}") - - if content_length > 0: - lnk['length'] = content_length - - collection['links'].append(lnk) - - # TODO: provide translations - LOGGER.debug('Adding JSON and HTML link relations') - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'root', - 'title': 'The landing page of this server as JSON', - 'href': f"{self.base_url}?f={F_JSON}" - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'root', - 'title': 'The landing page of this server as HTML', - 'href': f"{self.base_url}?f={F_HTML}" - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{self.get_collections_url()}/{k}?f={F_JSON}' - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{self.get_collections_url()}/{k}?f={F_JSONLD}' - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{self.get_collections_url()}/{k}?f={F_HTML}' - }) - - if collection_data_type in ['feature', 'coverage', 'record']: - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': f'{OGC_RELTYPES_BASE}/schema', - 'title': 'Schema of collection in JSON', - 'href': f'{self.get_collections_url()}/{k}/schema?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': f'{OGC_RELTYPES_BASE}/schema', - 'title': 'Schema of collection in HTML', - 'href': f'{self.get_collections_url()}/{k}/schema?f={F_HTML}' # noqa - }) - - if is_vector_tile or collection_data_type in ['feature', 'record']: - # TODO: translate - collection['itemType'] = collection_data_type - LOGGER.debug('Adding feature/record based links') - collection['links'].append({ - 'type': 'application/schema+json', - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', - 'title': 'Queryables for this collection as JSON', - 'href': f'{self.get_collections_url()}/{k}/queryables?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', - 'title': 'Queryables for this collection as HTML', - 'href': f'{self.get_collections_url()}/{k}/queryables?f={F_HTML}' # noqa - }) - collection['links'].append({ - 'type': 'application/geo+json', - 'rel': 'items', - 'title': 'items as GeoJSON', - 'href': f'{self.get_collections_url()}/{k}/items?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': 'items', - 'title': 'items as RDF (GeoJSON-LD)', - 'href': f'{self.get_collections_url()}/{k}/items?f={F_JSONLD}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'items', - 'title': 'Items as HTML', - 'href': f'{self.get_collections_url()}/{k}/items?f={F_HTML}' # noqa - }) - - # OAPIF Part 2 - list supported CRSs and StorageCRS - if collection_data_type == 'feature': - collection['crs'] = get_supported_crs_list(collection_data, DEFAULT_CRS_LIST) # noqa - collection['storageCRS'] = collection_data.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa - if 'storage_crs_coordinate_epoch' in collection_data: - collection['storageCrsCoordinateEpoch'] = collection_data.get('storage_crs_coordinate_epoch') # noqa - - elif collection_data_type == 'coverage': - # TODO: translate - LOGGER.debug('Adding coverage based links') - collection['links'].append({ - 'type': 'application/prs.coverage+json', - 'rel': f'{OGC_RELTYPES_BASE}/coverage', - 'title': 'Coverage data', - 'href': f'{self.get_collections_url()}/{k}/coverage?f={F_JSON}' # noqa - }) - if collection_data_format is not None: - collection['links'].append({ - 'type': collection_data_format['mimetype'], - 'rel': f'{OGC_RELTYPES_BASE}/coverage', - 'title': f"Coverage data as {collection_data_format['name']}", # noqa - 'href': f"{self.get_collections_url()}/{k}/coverage?f={collection_data_format['name']}" # noqa - }) - if dataset is not None: - LOGGER.debug('Creating extended coverage metadata') - try: - provider_def = get_provider_by_type( - self.config['resources'][k]['providers'], - 'coverage') - p = load_plugin('provider', provider_def) - except ProviderConnectionError: - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, - headers, request.format, - 'NoApplicableCode', msg) - except ProviderTypeError: - pass - else: - collection['extent']['spatial']['grid'] = [{ - 'cellsCount': p._coverage_properties['width'], - 'resolution': p._coverage_properties['resx'] - }, { - 'cellsCount': p._coverage_properties['height'], - 'resolution': p._coverage_properties['resy'] - }] - - try: - tile = get_provider_by_type(v['providers'], 'tile') - p = load_plugin('provider', tile) - except ProviderConnectionError: - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, - headers, request.format, - 'NoApplicableCode', msg) - except ProviderTypeError: - tile = None - - if tile: - # TODO: translate - - LOGGER.debug('Adding tile links') - collection['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa - 'title': 'Tiles as JSON', - 'href': f'{self.get_collections_url()}/{k}/tiles?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa - 'title': 'Tiles as HTML', - 'href': f'{self.get_collections_url()}/{k}/tiles?f={F_HTML}' # noqa - }) - - try: - map_ = get_provider_by_type(v['providers'], 'map') - except ProviderTypeError: - map_ = None - - if map_: - LOGGER.debug('Adding map links') - - map_mimetype = map_['format']['mimetype'] - map_format = map_['format']['name'] - - collection['links'].append({ - 'type': map_mimetype, - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/map', - 'title': f'Map as {map_format}', - 'href': f"{self.get_collections_url()}/{k}/map?f={map_format}" # noqa - }) - - try: - edr = get_provider_by_type(v['providers'], 'edr') - p = load_plugin('provider', edr) - except ProviderConnectionError: - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'NoApplicableCode', msg) - except ProviderTypeError: - edr = None - - if edr: - # TODO: translate - LOGGER.debug('Adding EDR links') - parameters = p.get_fields() - if parameters: - collection['parameter_names'] = {} - for key, value in parameters.items(): - collection['parameter_names'][key] = { - 'id': key, - 'type': 'Parameter', - 'name': value['title'], - 'unit': { - 'label': { - 'en': value['title'] - }, - 'symbol': { - 'value': value['x-ogc-unit'], - 'type': 'http://www.opengis.net/def/uom/UCUM/' # noqa - } - } - } - - for qt in p.get_query_types(): - collection['links'].append({ - 'type': 'application/json', - 'rel': 'data', - 'title': f'{qt} query for this collection as JSON', - 'href': f'{self.get_collections_url()}/{k}/{qt}?f={F_JSON}' # noqa - }) - collection['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'data', - 'title': f'{qt} query for this collection as HTML', - 'href': f'{self.get_collections_url()}/{k}/{qt}?f={F_HTML}' # noqa - }) - - if dataset is not None and k == dataset: - fcm = collection - break - - fcm['collections'].append(collection) - - if dataset is None: - # TODO: translate - fcm['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{self.get_collections_url()}?f={F_JSON}' - }) - fcm['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{self.get_collections_url()}?f={F_JSONLD}' - }) - fcm['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{self.get_collections_url()}?f={F_HTML}' - }) - - if request.format == F_HTML: # render - fcm['collections_path'] = self.get_collections_url() - if dataset is not None: - content = render_j2_template(self.tpl_config, - 'collections/collection.html', - fcm, request.locale) - else: - content = render_j2_template(self.tpl_config, - 'collections/index.html', fcm, - request.locale) - - return headers, HTTPStatus.OK, content - - if request.format == F_JSONLD: - jsonld = self.fcmld.copy() - if dataset is not None: - jsonld['dataset'] = jsonldify_collection(self, fcm, - request.locale) - else: - jsonld['dataset'] = [ - jsonldify_collection(self, c, request.locale) - for c in fcm.get('collections', []) - ] - return headers, HTTPStatus.OK, to_json(jsonld, self.pretty_print) - - return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def get_collection_schema( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Returns a collection schema - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection schema') - try: - LOGGER.debug('Loading feature provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'feature')) - except ProviderTypeError: - try: - LOGGER.debug('Loading coverage provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage')) # noqa - except ProviderTypeError: - LOGGER.debug('Loading record provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'record')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - schema = { - 'type': 'object', - 'title': l10n.translate( - self.config['resources'][dataset]['title'], request.locale), - 'properties': {}, - '$schema': 'http://json-schema.org/draft/2019-09/schema', - '$id': f'{self.get_collections_url()}/{dataset}/schema' - } - - if p.type != 'coverage': - schema['properties']['geometry'] = { - '$ref': 'https://geojson.org/schema/Geometry.json', - 'x-ogc-role': 'primary-geometry' - } - - for k, v in p.fields.items(): - schema['properties'][k] = v - - if k == p.id_field: - schema['properties'][k]['x-ogc-role'] = 'id' - if k == p.time_field: - schema['properties'][k]['x-ogc-role'] = 'primary-instant' - - if request.format == F_HTML: # render - schema['title'] = l10n.translate( - self.config['resources'][dataset]['title'], request.locale) - - schema['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/schema.html', - schema, request.locale) - - return headers, HTTPStatus.OK, content - - headers['Content-Type'] = 'application/schema+json' - - return headers, HTTPStatus.OK, to_json(schema, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def get_collection_queryables(self, request: Union[APIRequest, Any], - dataset=None) -> Tuple[dict, int, str]: - """ - Provide collection queryables - - :param request: A request object - :param dataset: name of collection - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection queryables') - try: - LOGGER.debug('Loading feature provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'feature')) - except ProviderTypeError: - try: - LOGGER.debug('Loading coverage provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage')) # noqa - except ProviderTypeError: - LOGGER.debug('Loading record provider') - p = load_plugin('provider', get_provider_by_type( - self.config['resources'][dataset]['providers'], 'record')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - queryables = { - 'type': 'object', - 'title': l10n.translate( - self.config['resources'][dataset]['title'], request.locale), - 'properties': {}, - '$schema': 'http://json-schema.org/draft/2019-09/schema', - '$id': f'{self.get_collections_url()}/{dataset}/queryables' - } - - if p.fields: - queryables['properties']['geometry'] = { - '$ref': 'https://geojson.org/schema/Geometry.json', - 'x-ogc-role': 'primary-geometry' - } - - for k, v in p.fields.items(): - show_field = False - if p.properties: - if k in p.properties: - show_field = True - else: - show_field = True - - if show_field: - queryables['properties'][k] = { - 'title': k, - 'type': v['type'] - } - if 'values' in v: - queryables['properties'][k]['enum'] = v['values'] - - if k == p.id_field: - queryables['properties'][k]['x-ogc-role'] = 'id' - if k == p.time_field: - queryables['properties'][k]['x-ogc-role'] = 'primary-instant' # noqa - - if request.format == F_HTML: # render - queryables['title'] = l10n.translate( - self.config['resources'][dataset]['title'], request.locale) - - queryables['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/queryables.html', - queryables, request.locale) - - return headers, HTTPStatus.OK, content - - headers['Content-Type'] = 'application/schema+json' - - return headers, HTTPStatus.OK, to_json(queryables, self.pretty_print) - - @gzip - @pre_process - def get_collection_items( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Queries collection - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - properties = [] - reserved_fieldnames = ['bbox', 'bbox-crs', 'crs', 'f', 'lang', 'limit', - 'offset', 'resulttype', 'datetime', 'sortby', - 'properties', 'skipGeometry', 'q', - 'filter', 'filter-lang', 'filter-crs'] - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing offset parameter') - try: - offset = int(request.params.get('offset')) - if offset < 0: - msg = 'offset value should be positive or zero' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - offset = 0 - except ValueError: - msg = 'offset value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - # TODO: We should do more validation, against the min and max - # allowed by the server configuration - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - limit = int(self.config['server']['limit']) - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - resulttype = request.params.get('resulttype') or 'results' - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('processing q parameter') - q = request.params.get('q') or None - - LOGGER.debug('Loading provider') - - provider_def = None - try: - provider_type = 'feature' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_type = 'record' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - crs_transform_spec = None - if provider_type == 'feature': - # crs query parameter is only available for OGC API - Features - # right now, not for OGC API - Records. - LOGGER.debug('Processing crs parameter') - query_crs_uri = request.params.get('crs') - try: - crs_transform_spec = self._create_crs_transform_spec( - provider_def, query_crs_uri, - ) - except (ValueError, CRSError) as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - self._set_content_crs_header(headers, provider_def, query_crs_uri) - - LOGGER.debug('Processing bbox-crs parameter') - bbox_crs = request.params.get('bbox-crs') - if bbox_crs is not None: - # Validate bbox-crs parameter - if len(bbox) == 0: - msg = 'bbox-crs specified without bbox parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - if len(bbox_crs) == 0: - msg = 'bbox-crs specified but is empty' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - supported_crs_list = get_supported_crs_list(provider_def, DEFAULT_CRS_LIST) # noqa - if bbox_crs not in supported_crs_list: - msg = f'bbox-crs {bbox_crs} not supported for this collection' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - elif len(bbox) > 0: - # bbox but no bbox-crs parm: assume bbox is in default CRS - bbox_crs = DEFAULT_CRS - - # Transform bbox to storageCRS - # when bbox-crs different from storageCRS. - if len(bbox) > 0: - try: - # Get a pyproj CRS instance for the Collection's Storage CRS - storage_crs = provider_def.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa - - # Do the (optional) Transform to the Storage CRS - bbox = transform_bbox(bbox, bbox_crs, storage_crs) - except CRSError as e: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', str(e)) - - LOGGER.debug('processing property parameters') - for k, v in request.params.items(): - if k not in reserved_fieldnames and k in list(p.fields.keys()): - LOGGER.debug(f'Adding property filter {k}={v}') - properties.append((k, v)) - - LOGGER.debug('processing sort parameter') - val = request.params.get('sortby') - - if val is not None: - sortby = [] - sorts = val.split(',') - for s in sorts: - prop = s - order = '+' - if s[0] in ['+', '-']: - order = s[0] - prop = s[1:] - - if prop not in p.fields.keys(): - msg = 'bad sort property' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - sortby.append({'property': prop, 'order': order}) - else: - sortby = [] - - LOGGER.debug('processing properties parameter') - val = request.params.get('properties') - - if val is not None: - select_properties = val.split(',') - properties_to_check = set(p.properties) | set(p.fields.keys()) - - if (len(list(set(select_properties) - - set(properties_to_check))) > 0): - msg = 'unknown properties specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - select_properties = [] - - LOGGER.debug('processing skipGeometry parameter') - val = request.params.get('skipGeometry') - if val is not None: - skip_geometry = str2bool(val) - else: - skip_geometry = False - - LOGGER.debug('Processing filter-crs parameter') - filter_crs_uri = request.params.get('filter-crs', DEFAULT_CRS) - LOGGER.debug('processing filter parameter') - cql_text = request.params.get('filter') - if cql_text is not None: - try: - filter_ = parse_ecql_text(cql_text) - filter_ = modify_pygeofilter( - filter_, - filter_crs_uri=filter_crs_uri, - storage_crs_uri=provider_def.get('storage_crs'), - geometry_column_name=provider_def.get('geom_field'), - ) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {cql_text}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - filter_ = None - - LOGGER.debug('Processing filter-lang parameter') - filter_lang = request.params.get('filter-lang') - # Currently only cql-text is handled, but it is optional - if filter_lang not in [None, 'cql-text']: - msg = 'Invalid filter language' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - # Get provider locale (if any) - prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) - - LOGGER.debug('Querying provider') - LOGGER.debug(f'offset: {offset}') - LOGGER.debug(f'limit: {limit}') - LOGGER.debug(f'resulttype: {resulttype}') - LOGGER.debug(f'sortby: {sortby}') - LOGGER.debug(f'bbox: {bbox}') - if provider_type == 'feature': - LOGGER.debug(f'crs: {query_crs_uri}') - LOGGER.debug(f'datetime: {datetime_}') - LOGGER.debug(f'properties: {properties}') - LOGGER.debug(f'select properties: {select_properties}') - LOGGER.debug(f'skipGeometry: {skip_geometry}') - LOGGER.debug(f'language: {prv_locale}') - LOGGER.debug(f'q: {q}') - LOGGER.debug(f'cql_text: {cql_text}') - LOGGER.debug(f'filter_: {filter_}') - LOGGER.debug(f'filter-lang: {filter_lang}') - LOGGER.debug(f'filter-crs: {filter_crs_uri}') - - try: - content = p.query(offset=offset, limit=limit, - resulttype=resulttype, bbox=bbox, - datetime_=datetime_, properties=properties, - sortby=sortby, skip_geometry=skip_geometry, - select_properties=select_properties, - crs_transform_spec=crs_transform_spec, - q=q, language=prv_locale, filterq=filter_) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - serialized_query_params = '' - for k, v in request.params.items(): - if k not in ('f', 'offset'): - serialized_query_params += '&' - serialized_query_params += urllib.parse.quote(k, safe='') - serialized_query_params += '=' - serialized_query_params += urllib.parse.quote(str(v), safe=',') - - # TODO: translate titles - uri = f'{self.get_collections_url()}/{dataset}/items' - content['links'] = [{ - 'type': 'application/geo+json', - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as GeoJSON', - 'href': f'{uri}?f={F_JSON}{serialized_query_params}' - }, { - 'rel': request.get_linkrel(F_JSONLD), - 'type': FORMAT_TYPES[F_JSONLD], - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{uri}?f={F_HTML}{serialized_query_params}' - }] - - if offset > 0: - prev = max(0, offset - limit) - content['links'].append( - { - 'type': 'application/geo+json', - 'rel': 'prev', - 'title': 'items (prev)', - 'href': f'{uri}?offset={prev}{serialized_query_params}' - }) - - if 'numberMatched' in content: - if content['numberMatched'] > (limit + offset): - next_ = offset + limit - next_href = f'{uri}?offset={next_}{serialized_query_params}' - content['links'].append( - { - 'type': 'application/geo+json', - 'rel': 'next', - 'title': 'items (next)', - 'href': next_href - }) - - content['links'].append( - { - 'type': FORMAT_TYPES[F_JSON], - 'title': l10n.translate( - collections[dataset]['title'], request.locale), - 'rel': 'collection', - 'href': uri - }) - - content['timeStamp'] = datetime.utcnow().strftime( - '%Y-%m-%dT%H:%M:%S.%fZ') - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - if request.format == F_HTML: # render - # For constructing proper URIs to items - - content['items_path'] = uri - content['dataset_path'] = '/'.join(uri.split('/')[:-1]) - content['collections_path'] = self.get_collections_url() - - content['offset'] = offset - - content['id_field'] = p.id_field - if p.uri_field is not None: - content['uri_field'] = p.uri_field - if p.title_field is not None: - content['title_field'] = l10n.translate(p.title_field, - request.locale) - # If title exists, use it as id in html templates - content['id_field'] = content['title_field'] - content = render_j2_template(self.tpl_config, - 'collections/items/index.html', - content, request.locale) - return headers, HTTPStatus.OK, content - elif request.format == 'csv': # render - formatter = load_plugin('formatter', - {'name': 'CSV', 'geom': True}) - - try: - content = formatter.write( - data=content, - options={ - 'provider_def': get_provider_by_type( - collections[dataset]['providers'], - 'feature') - } - ) - except FormatterSerializationError as err: - LOGGER.error(err) - msg = 'Error serializing output' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, - 'NoApplicableCode', msg) - - headers['Content-Type'] = formatter.mimetype - - if p.filename is None: - filename = f'{dataset}.csv' - else: - filename = f'{p.filename}' - - cd = f'attachment; filename="{filename}"' - headers['Content-Disposition'] = cd - - return headers, HTTPStatus.OK, content - - elif request.format == F_JSONLD: - content = geojson2jsonld( - self, content, dataset, id_field=(p.uri_field or 'id') - ) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - def post_collection_items( - self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Queries collection or filter an item - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - request_headers = request.headers - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - properties = [] - reserved_fieldnames = ['bbox', 'f', 'limit', 'offset', - 'resulttype', 'datetime', 'sortby', - 'properties', 'skipGeometry', 'q', - 'filter-lang', 'filter-crs'] - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Invalid collection' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing offset parameter') - try: - offset = int(request.params.get('offset')) - if offset < 0: - msg = 'offset value should be positive or zero' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - offset = 0 - except ValueError: - msg = 'offset value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - # TODO: We should do more validation, against the min and max - # allowed by the server configuration - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except TypeError as err: - LOGGER.warning(err) - limit = int(self.config['server']['limit']) - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - resulttype = request.params.get('resulttype') or 'results' - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('processing q parameter') - val = request.params.get('q') - - q = None - if val is not None: - q = val - - LOGGER.debug('Loading provider') - - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'feature') - except ProviderTypeError: - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'record') - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'NoApplicableCode', msg) - - try: - p = load_plugin('provider', provider_def) - except ProviderGenericError as err: - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('processing property parameters') - for k, v in request.params.items(): - if k not in reserved_fieldnames and k not in p.fields.keys(): - msg = f'unknown query parameter: {k}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - elif k not in reserved_fieldnames and k in p.fields.keys(): - LOGGER.debug(f'Add property filter {k}={v}') - properties.append((k, v)) - - LOGGER.debug('processing sort parameter') - val = request.params.get('sortby') - - if val is not None: - sortby = [] - sorts = val.split(',') - for s in sorts: - prop = s - order = '+' - if s[0] in ['+', '-']: - order = s[0] - prop = s[1:] - - if prop not in p.fields.keys(): - msg = 'bad sort property' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - sortby.append({'property': prop, 'order': order}) - else: - sortby = [] - - LOGGER.debug('processing properties parameter') - val = request.params.get('properties') - - if val is not None: - select_properties = val.split(',') - properties_to_check = set(p.properties) | set(p.fields.keys()) - - if (len(list(set(select_properties) - - set(properties_to_check))) > 0): - msg = 'unknown properties specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - select_properties = [] - - LOGGER.debug('processing skipGeometry parameter') - val = request.params.get('skipGeometry') - if val is not None: - skip_geometry = str2bool(val) - else: - skip_geometry = False - - LOGGER.debug('Processing filter-crs parameter') - filter_crs = request.params.get('filter-crs', DEFAULT_CRS) - LOGGER.debug('Processing filter-lang parameter') - filter_lang = request.params.get('filter-lang') - if filter_lang != 'cql-json': # @TODO add check from the configuration - msg = 'Invalid filter language' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Querying provider') - LOGGER.debug(f'offset: {offset}') - LOGGER.debug(f'limit: {limit}') - LOGGER.debug(f'resulttype: {resulttype}') - LOGGER.debug(f'sortby: {sortby}') - LOGGER.debug(f'bbox: {bbox}') - LOGGER.debug(f'datetime: {datetime_}') - LOGGER.debug(f'properties: {select_properties}') - LOGGER.debug(f'skipGeometry: {skip_geometry}') - LOGGER.debug(f'q: {q}') - LOGGER.debug(f'filter-lang: {filter_lang}') - LOGGER.debug(f'filter-crs: {filter_crs}') - - LOGGER.debug('Processing headers') - - LOGGER.debug('Processing request content-type header') - if (request_headers.get( - 'Content-Type') or request_headers.get( - 'content-type')) != 'application/query-cql-json': - msg = ('Invalid body content-type') - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidHeaderValue', msg) - - LOGGER.debug('Processing body') - - if not request.data: - msg = 'missing request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'MissingParameterValue', msg) - - filter_ = None - try: - # Parse bytes data, if applicable - data = request.data.decode() - LOGGER.debug(data) - except UnicodeDecodeError as err: - LOGGER.error(err) - msg = 'Unicode error in data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - # FIXME: remove testing backend in use once CQL support is normalized - if p.name == 'PostgreSQL': - LOGGER.debug('processing PostgreSQL CQL_JSON data') - try: - filter_ = parse_cql_json(data) - filter_ = modify_pygeofilter( - filter_, - filter_crs_uri=filter_crs, - storage_crs_uri=provider_def.get('storage_crs'), - geometry_column_name=provider_def.get('geom_field') - ) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {data}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - else: - LOGGER.debug('processing Elasticsearch CQL_JSON data') - try: - filter_ = CQLModel.parse_raw(data) - except Exception as err: - LOGGER.error(err) - msg = f'Bad CQL string : {data}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - try: - content = p.query(offset=offset, limit=limit, - resulttype=resulttype, bbox=bbox, - datetime_=datetime_, properties=properties, - sortby=sortby, - select_properties=select_properties, - skip_geometry=skip_geometry, - q=q, - filterq=filter_) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - def manage_collection_item( - self, request: Union[APIRequest, Any], - action, dataset, identifier=None) -> Tuple[dict, int, str]: - """ - Adds an item to a collection - - :param request: A request object - :param action: an action among 'create', 'update', 'delete', 'options' - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'feature') - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_def = get_provider_by_type( - collections[dataset]['providers'], 'record') - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action == 'options': - headers['Allow'] = 'HEAD, GET' - if p.editable: - if identifier is None: - headers['Allow'] += ', POST' - else: - headers['Allow'] += ', PUT, DELETE' - return headers, HTTPStatus.OK, '' - - if not p.editable: - msg = 'Collection is not editable' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action in ['create', 'update'] and not request.data: - msg = 'No data found' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if action == 'create': - LOGGER.debug('Creating item') - try: - identifier = p.create(request.data) - except TypeError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - headers['Location'] = f'{self.get_collections_url()}/{dataset}/items/{identifier}' # noqa - - return headers, HTTPStatus.CREATED, '' - - if action == 'update': - LOGGER.debug('Updating item') - try: - _ = p.update(identifier, request.data) - except TypeError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.NO_CONTENT, '' - - if action == 'delete': - LOGGER.debug('Deleting item') - try: - _ = p.delete(identifier) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - return headers, HTTPStatus.OK, '' - - @gzip - @pre_process - def get_collection_item(self, request: Union[APIRequest, Any], - dataset, identifier) -> Tuple[dict, int, str]: - """ - Get a single collection item - - :param request: A request object - :param dataset: dataset name - :param identifier: item identifier - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - # Set Content-Language to system locale until provider locale - # has been determined - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - LOGGER.debug('Processing query parameters') - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - - try: - provider_type = 'feature' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - try: - provider_type = 'record' - provider_def = get_provider_by_type( - collections[dataset]['providers'], provider_type) - p = load_plugin('provider', provider_def) - except ProviderTypeError: - msg = 'Invalid provider type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - crs_transform_spec = None - if provider_type == 'feature': - # crs query parameter is only available for OGC API - Features - # right now, not for OGC API - Records. - LOGGER.debug('Processing crs parameter') - query_crs_uri = request.params.get('crs') - try: - crs_transform_spec = self._create_crs_transform_spec( - provider_def, query_crs_uri, - ) - except (ValueError, CRSError) as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - self._set_content_crs_header(headers, provider_def, query_crs_uri) - - # Get provider language (if any) - prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) - - try: - LOGGER.debug(f'Fetching id {identifier}') - content = p.get( - identifier, - language=prv_locale, - crs_transform_spec=crs_transform_spec, - ) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if content is None: - msg = 'identifier not found' - return self.get_exception(HTTPStatus.BAD_REQUEST, headers, - request.format, 'NotFound', msg) - - uri = content['properties'].get(p.uri_field) if p.uri_field else \ - f'{self.get_collections_url()}/{dataset}/items/{identifier}' - - if 'links' not in content: - content['links'] = [] - - content['links'].extend([{ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'root', - 'title': 'The landing page of this server as JSON', - 'href': f"{self.base_url}?f={F_JSON}" - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'root', - 'title': 'The landing page of this server as HTML', - 'href': f"{self.base_url}?f={F_HTML}" - }, { - 'rel': request.get_linkrel(F_JSON), - 'type': 'application/geo+json', - 'title': 'This document as GeoJSON', - 'href': f'{uri}?f={F_JSON}' - }, { - 'rel': request.get_linkrel(F_JSONLD), - 'type': FORMAT_TYPES[F_JSONLD], - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{uri}?f={F_JSONLD}' - }, { - 'rel': request.get_linkrel(F_HTML), - 'type': FORMAT_TYPES[F_HTML], - 'title': 'This document as HTML', - 'href': f'{uri}?f={F_HTML}' - }, { - 'rel': 'collection', - 'type': FORMAT_TYPES[F_JSON], - 'title': l10n.translate(collections[dataset]['title'], - request.locale), - 'href': f'{self.get_collections_url()}/{dataset}' - }]) - - link_request_format = ( - request.format if request.format is not None else F_JSON - ) - if 'prev' in content: - content['links'].append({ - 'rel': 'prev', - 'type': FORMAT_TYPES[link_request_format], - 'href': f"{self.get_collections_url()}/{dataset}/items/{content['prev']}?f={link_request_format}" # noqa - }) - if 'next' in content: - content['links'].append({ - 'rel': 'next', - 'type': FORMAT_TYPES[link_request_format], - 'href': f"{self.get_collections_url()}/{dataset}/items/{content['next']}?f={link_request_format}" # noqa - }) - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - if request.format == F_HTML: # render - content['title'] = l10n.translate(collections[dataset]['title'], - request.locale) - content['id_field'] = p.id_field - if p.uri_field is not None: - content['uri_field'] = p.uri_field - if p.title_field is not None: - content['title_field'] = l10n.translate(p.title_field, - request.locale) - content['collections_path'] = self.get_collections_url() - - content = render_j2_template(self.tpl_config, - 'collections/items/item.html', - content, request.locale) - return headers, HTTPStatus.OK, content - - elif request.format == F_JSONLD: - content = geojson2jsonld( - self, content, dataset, uri, (p.uri_field or 'id') - ) - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @pre_process - @jsonldify - def get_collection_coverage(self, request: Union[APIRequest, Any], - dataset) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection coverage - - :param request: A request object - :param dataset: dataset name - - :returns: tuple of headers, status code, content - """ - - query_args = {} - format_ = request.format or F_JSON - - # Force response content type and language (en-US only) headers - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'coverage') - - p = load_plugin('provider', collection_def) - except KeyError: - msg = 'collection does not exist' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, format_, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('Processing bbox parameter') - - bbox = request.params.get('bbox') - - if bbox is None: - bbox = [] - else: - try: - bbox = validate_bbox(bbox) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, format_, - 'InvalidParameterValue', msg) - - query_args['bbox'] = bbox - - LOGGER.debug('Processing bbox-crs parameter') - - bbox_crs = request.params.get('bbox-crs') - if bbox_crs is not None: - query_args['bbox_crs'] = bbox_crs - - LOGGER.debug('Processing datetime parameter') - - datetime_ = request.params.get('datetime') - - try: - datetime_ = validate_datetime( - self.config['resources'][dataset]['extents'], datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - query_args['datetime_'] = datetime_ - query_args['format_'] = format_ - - properties = request.params.get('properties') - if properties: - LOGGER.debug('Processing properties parameter') - query_args['properties'] = [rs for - rs in properties.split(',') if rs] - LOGGER.debug(f"Fields: {query_args['properties']}") - - for a in query_args['properties']: - if a not in p.fields: - msg = 'Invalid field specified' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - if 'subset' in request.params: - LOGGER.debug('Processing subset parameter') - try: - subsets = validate_subset(request.params['subset'] or '') - except (AttributeError, ValueError) as err: - msg = f'Invalid subset: {err}' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - if not set(subsets.keys()).issubset(p.axes): - msg = 'Invalid axis name' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - - query_args['subsets'] = subsets - LOGGER.debug(f"Subsets: {query_args['subsets']}") - - LOGGER.debug('Querying coverage') - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - if format_ == mt: # native format - if p.filename is not None: - cd = f'attachment; filename="{p.filename}"' - headers['Content-Disposition'] = cd - - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - elif format_ == F_JSON: - headers['Content-Type'] = 'application/prs.coverage+json' - return headers, HTTPStatus.OK, to_json(data, self.pretty_print) - else: - return self.get_format_exception(request) - - @gzip - @pre_process - @jsonldify - def get_collection_tiles(self, request: Union[APIRequest, Any], - dataset=None) -> Tuple[dict, int, str]: - """ - Provide collection tiles - - :param request: A request object - :param dataset: name of collection - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection tiles') - LOGGER.debug('Loading provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - except (KeyError, ProviderTypeError): - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - tiles = { - 'links': [], - 'tilesets': [] - } - - tiles['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSON}' - }) - tiles['links'].append({ - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' # noqa - }) - tiles['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{self.get_collections_url()}/{dataset}/tiles?f={F_HTML}' - }) - - tile_services = p.get_tiles_service( - baseurl=self.base_url, - servicepath=f'{self.get_collections_url()}/{dataset}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={p.format_type}' # noqa - ) - - for service in tile_services['links']: - tiles['links'].append(service) - - tiling_schemes = p.get_tiling_schemes() - - for matrix in tiling_schemes: - tile_matrix = { - 'title': dataset, - 'tileMatrixSetURI': matrix.tileMatrixSetURI, - 'crs': matrix.crs, - 'dataType': 'vector', - 'links': [] - } - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme', - 'title': f'{matrix.tileMatrixSet} TileMatrixSet definition (as {F_JSON})', # noqa - 'href': f'{self.base_url}/TileMatrixSets/{matrix.tileMatrixSet}?f={F_JSON}' # noqa - }) - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_JSON}', - 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_JSON}' # noqa - }) - tile_matrix['links'].append({ - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_HTML}', - 'href': f'{self.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_HTML}' # noqa - }) - - tiles['tilesets'].append(tile_matrix) - - if request.format == F_HTML: # render - tiles['id'] = dataset - tiles['title'] = l10n.translate( - self.config['resources'][dataset]['title'], SYSTEM_LOCALE) - tiles['tilesets'] = [ - scheme.tileMatrixSet for scheme in p.get_tiling_schemes()] - tiles['bounds'] = \ - self.config['resources'][dataset]['extents']['spatial']['bbox'] - tiles['minzoom'] = p.options['zoom']['min'] - tiles['maxzoom'] = p.options['zoom']['max'] - tiles['collections_path'] = self.get_collections_url() - tiles['tile_type'] = p.tile_type - - content = render_j2_template(self.tpl_config, - 'collections/tiles/index.html', tiles, - request.locale) - - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(tiles, self.pretty_print) - - @pre_process - def get_collection_tiles_data( - self, request: Union[APIRequest, Any], - dataset=None, matrix_id=None, - z_idx=None, y_idx=None, x_idx=None) -> Tuple[dict, int, str]: - """ - Get collection items tiles - - :param request: A request object - :param dataset: dataset name - :param matrix_id: matrix identifier - :param z_idx: z index - :param y_idx: y index - :param x_idx: x index - - :returns: tuple of headers, status code, content - """ - - format_ = request.format - if not format_: - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - LOGGER.debug('Processing tiles') - - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Loading tile provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - - format_ = p.format_type - headers['Content-Type'] = format_ - - LOGGER.debug(f'Fetching tileset id {matrix_id} and tile {z_idx}/{y_idx}/{x_idx}') # noqa - content = p.get_tiles(layer=p.get_layer(), tileset=matrix_id, - z=z_idx, y=y_idx, x=x_idx, format_=format_) - if content is None: - msg = 'identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, format_, 'NotFound', msg) - else: - return headers, HTTPStatus.OK, content - - # @TODO: figure out if the spec requires to return json errors - except KeyError: - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, format_, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - @gzip - @pre_process - @jsonldify - def get_collection_tiles_metadata( - self, request: Union[APIRequest, Any], - dataset=None, matrix_id=None) -> Tuple[dict, int, str]: - """ - Get collection items tiles - - :param request: A request object - :param dataset: dataset name - :param matrix_id: matrix identifier - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid([TilesMetadataFormat.TILEJSON]): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if any([dataset is None, - dataset not in self.config['resources'].keys()]): - - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Creating collection tiles') - LOGGER.debug('Loading provider') - try: - t = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'tile') - p = load_plugin('provider', t) - except KeyError: - msg = 'Invalid collection tiles' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - # Get provider language (if any) - prv_locale = l10n.get_plugin_locale(t, request.raw_locale) - - # Set response language to requested provider locale - # (if it supports language) and/or otherwise the requested pygeoapi - # locale (or fallback default locale) - l10n.set_response_language(headers, prv_locale, request.locale) - - tiles_metadata = p.get_metadata( - dataset=dataset, server_url=self.base_url, - layer=p.get_layer(), tileset=matrix_id, - metadata_format=request._format, title=l10n.translate( - self.config['resources'][dataset]['title'], - request.locale), - description=l10n.translate( - self.config['resources'][dataset]['description'], - request.locale), - language=prv_locale) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'collections/tiles/metadata.html', - tiles_metadata, request.locale) - - return headers, HTTPStatus.OK, content - else: - return headers, HTTPStatus.OK, tiles_metadata - - @gzip - @pre_process - def get_collection_map(self, request: Union[APIRequest, Any], - dataset, style=None) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection map - - :param request: A request object - :param dataset: dataset name - :param style: style name - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - query_args = { - 'crs': 'CRS84' - } - - format_ = request.format or 'png' - headers = request.get_response_headers(**self.api_headers) - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'map') - - p = load_plugin('provider', collection_def) - except KeyError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'collection does not exist' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.NOT_FOUND, to_json( - exception, self.pretty_print) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - query_args['format_'] = request.params.get('f', 'png') - query_args['style'] = style - query_args['crs'] = request.params.get('bbox-crs', 4326) - query_args['transparent'] = request.params.get('transparent', True) - - try: - query_args['width'] = int(request.params.get('width', 500)) - query_args['height'] = int(request.params.get('height', 300)) - except ValueError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid width/height' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - - LOGGER.debug('Processing bbox parameter') - try: - bbox = request.params.get('bbox').split(',') - if len(bbox) != 4: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'bbox values should be minx,miny,maxx,maxy' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - except AttributeError: - bbox = self.config['resources'][dataset]['extents']['spatial']['bbox'] # noqa - try: - query_args['bbox'] = [float(c) for c in bbox] - except ValueError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'bbox values must be numbers' - } - headers['Content-type'] = 'application/json' - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - exception, self.pretty_print) - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - query_args['datetime_'] = validate_datetime( - self.config['resources'][dataset]['extents'], datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Generating map') - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - - if format_ == mt: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - elif format_ in [None, 'html']: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - else: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid format parameter' - } - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - data, self.pretty_print) - - @gzip - def get_collection_map_legend( - self, request: Union[APIRequest, Any], - dataset, style=None) -> Tuple[dict, int, str]: - """ - Returns a subset of a collection map legend - - :param request: A request object - :param dataset: dataset name - :param style: style name - - :returns: tuple of headers, status code, content - """ - - format_ = 'png' - headers = request.get_response_headers(**self.api_headers) - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Loading provider') - try: - collection_def = get_provider_by_type( - self.config['resources'][dataset]['providers'], 'map') - - p = load_plugin('provider', collection_def) - except KeyError: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'collection does not exist' - } - LOGGER.error(exception) - return headers, HTTPStatus.NOT_FOUND, to_json( - exception, self.pretty_print) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - LOGGER.debug('Generating legend') - try: - data = p.get_legend(style, request.params.get('f', 'png')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - mt = collection_def['format']['name'] - - if format_ == mt: - headers['Content-Type'] = collection_def['format']['mimetype'] - return headers, HTTPStatus.OK, data - else: - exception = { - 'code': 'InvalidParameterValue', - 'description': 'invalid format parameter' - } - LOGGER.error(exception) - return headers, HTTPStatus.BAD_REQUEST, to_json( - data, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def describe_processes(self, request: Union[APIRequest, Any], - process=None) -> Tuple[dict, int, str]: - """ - Provide processes metadata - - :param request: A request object - :param process: process identifier, defaults to None to obtain - information about all processes - - :returns: tuple of headers, status code, content - """ - - processes = [] - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - if process is not None: - if process not in self.manager.processes.keys(): - msg = 'Identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchProcess', msg) - - if len(self.manager.processes) > 0: - if process is not None: - relevant_processes = [process] - else: - LOGGER.debug('Processing limit parameter') - try: - limit = int(request.params.get('limit')) - - if limit <= 0: - msg = 'limit value should be strictly positive' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - relevant_processes = list(self.manager.processes)[:limit] - except TypeError: - LOGGER.debug('returning all processes') - relevant_processes = self.manager.processes.keys() - except ValueError: - msg = 'limit value should be an integer' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - for key in relevant_processes: - p = self.manager.get_processor(key) - p2 = l10n.translate_struct(deepcopy(p.metadata), - request.locale) - p2['id'] = key - - if process is None: - p2.pop('inputs') - p2.pop('outputs') - p2.pop('example', None) - - p2['jobControlOptions'] = ['sync-execute'] - if self.manager.is_async: - p2['jobControlOptions'].append('async-execute') - - p2['outputTransmission'] = ['value'] - p2['links'] = p2.get('links', []) - - jobs_url = f"{self.base_url}/jobs" - process_url = f"{self.base_url}/processes/{key}" - - # TODO translation support - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'href': f'{process_url}?f={F_JSON}', - 'title': 'Process description as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'href': f'{process_url}?f={F_HTML}', - 'title': 'Process description as HTML', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_HTML], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', - 'href': f'{jobs_url}?f={F_HTML}', - 'title': 'jobs for this process as HTML', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', - 'href': f'{jobs_url}?f={F_JSON}', - 'title': 'jobs for this process as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - link = { - 'type': FORMAT_TYPES[F_JSON], - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/execute', - 'href': f'{process_url}/execution?f={F_JSON}', - 'title': 'Execution for this process as JSON', - 'hreflang': self.default_locale - } - p2['links'].append(link) - - processes.append(p2) - - if process is not None: - response = processes[0] - else: - process_url = f"{self.base_url}/processes" - response = { - 'processes': processes, - 'links': [{ - 'type': FORMAT_TYPES[F_JSON], - 'rel': request.get_linkrel(F_JSON), - 'title': 'This document as JSON', - 'href': f'{process_url}?f={F_JSON}' - }, { - 'type': FORMAT_TYPES[F_JSONLD], - 'rel': request.get_linkrel(F_JSONLD), - 'title': 'This document as RDF (JSON-LD)', - 'href': f'{process_url}?f={F_JSONLD}' - }, { - 'type': FORMAT_TYPES[F_HTML], - 'rel': request.get_linkrel(F_HTML), - 'title': 'This document as HTML', - 'href': f'{process_url}?f={F_HTML}' - }] - } - - if request.format == F_HTML: # render - if process is not None: - response = render_j2_template(self.tpl_config, - 'processes/process.html', - response, request.locale) - else: - response = render_j2_template(self.tpl_config, - 'processes/index.html', response, - request.locale) - - return headers, HTTPStatus.OK, response - - return headers, HTTPStatus.OK, to_json(response, self.pretty_print) - - @gzip - @pre_process - def get_jobs(self, request: Union[APIRequest, Any], - job_id=None) -> Tuple[dict, int, str]: - """ - Get process jobs - - :param request: A request object - :param job_id: id of job - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if job_id is None: - jobs = sorted(self.manager.get_jobs(), - key=lambda k: k['job_start_datetime'], - reverse=True) - else: - try: - jobs = [self.manager.get_job(job_id)] - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, - 'InvalidParameterValue', job_id) - - serialized_jobs = { - 'jobs': [], - 'links': [{ - 'href': f"{self.base_url}/jobs?f={F_HTML}", - 'rel': request.get_linkrel(F_HTML), - 'type': FORMAT_TYPES[F_HTML], - 'title': 'Jobs list as HTML' - }, { - 'href': f"{self.base_url}/jobs?f={F_JSON}", - 'rel': request.get_linkrel(F_JSON), - 'type': FORMAT_TYPES[F_JSON], - 'title': 'Jobs list as JSON' - }] - } - for job_ in jobs: - job2 = { - 'type': 'process', - 'processID': job_['process_id'], - 'jobID': job_['identifier'], - 'status': job_['status'], - 'message': job_['message'], - 'progress': job_['progress'], - 'parameters': job_.get('parameters'), - 'job_start_datetime': job_['job_start_datetime'], - 'job_end_datetime': job_['job_end_datetime'] - } - - # TODO: translate - if JobStatus[job_['status']] in ( - JobStatus.successful, JobStatus.running, JobStatus.accepted): - - job_result_url = f"{self.base_url}/jobs/{job_['identifier']}/results" # noqa - - job2['links'] = [{ - 'href': f'{job_result_url}?f={F_HTML}', - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', - 'type': FORMAT_TYPES[F_HTML], - 'title': f'results of job {job_id} as HTML' - }, { - 'href': f'{job_result_url}?f={F_JSON}', - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', - 'type': FORMAT_TYPES[F_JSON], - 'title': f'results of job {job_id} as JSON' - }] - - if job_['mimetype'] not in (FORMAT_TYPES[F_JSON], - FORMAT_TYPES[F_HTML]): - - job2['links'].append({ - 'href': job_result_url, - 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', # noqa - 'type': job_['mimetype'], - 'title': f"results of job {job_id} as {job_['mimetype']}" # noqa - }) - - serialized_jobs['jobs'].append(job2) - - if job_id is None: - j2_template = 'jobs/index.html' - else: - serialized_jobs = serialized_jobs['jobs'][0] - j2_template = 'jobs/job.html' - - if request.format == F_HTML: - data = { - 'jobs': serialized_jobs, - 'now': datetime.now(timezone.utc).strftime(DATETIME_FORMAT) - } - response = render_j2_template(self.tpl_config, j2_template, data, - request.locale) - return headers, HTTPStatus.OK, response - - return headers, HTTPStatus.OK, to_json(serialized_jobs, - self.pretty_print) - - @gzip - @pre_process - def execute_process(self, request: Union[APIRequest, Any], - process_id) -> Tuple[dict, int, str]: - """ - Execute process - - :param request: A request object - :param process_id: id of process - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - - # Responses are always in US English only - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - if process_id not in self.manager.processes: - msg = 'identifier not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchProcess', msg) - - data = request.data - if not data: - # TODO not all processes require input, e.g. time-dependent or - # random value generators - msg = 'missing request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'MissingParameterValue', msg) - - try: - # Parse bytes data, if applicable - data = data.decode() - LOGGER.debug(data) - except (UnicodeDecodeError, AttributeError): - pass - - try: - data = json.loads(data) - except (json.decoder.JSONDecodeError, TypeError) as err: - # Input does not appear to be valid JSON - LOGGER.error(err) - msg = 'invalid request data' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - data_dict = data.get('inputs', {}) - LOGGER.debug(data_dict) - - subscriber = None - subscriber_dict = data.get('subscriber') - if subscriber_dict: - try: - success_uri = subscriber_dict['successUri'] - except KeyError: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'MissingParameterValue', 'Missing successUri') - else: - subscriber = Subscriber( - # NOTE: successUri is mandatory according to the standard - success_uri=success_uri, - in_progress_uri=subscriber_dict.get('inProgressUri'), - failed_uri=subscriber_dict.get('failedUri'), - ) - - try: - execution_mode = RequestedProcessExecutionMode( - request.headers.get('Prefer', request.headers.get('prefer')) - ) - except ValueError: - execution_mode = None - try: - LOGGER.debug('Executing process') - result = self.manager.execute_process( - process_id, - data_dict, - execution_mode=execution_mode, - subscriber=subscriber, - ) - job_id, mime_type, outputs, status, additional_headers = result - headers.update(additional_headers or {}) - headers['Location'] = f'{self.base_url}/jobs/{job_id}' - except ProcessorExecuteError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, - request.format, err.ogc_exception_code, err.message) - - response = {} - if status == JobStatus.failed: - response = outputs - - if data.get('response', 'raw') == 'raw': - headers['Content-Type'] = mime_type - response = outputs - elif status not in (JobStatus.failed, JobStatus.accepted): - response['outputs'] = [outputs] - - if status == JobStatus.accepted: - http_status = HTTPStatus.CREATED - elif status == JobStatus.failed: - http_status = HTTPStatus.BAD_REQUEST - else: - http_status = HTTPStatus.OK - - if mime_type == 'application/json': - response2 = to_json(response, self.pretty_print) - else: - response2 = response - - return headers, http_status, response2 - - @gzip - @pre_process - def get_job_result(self, request: Union[APIRequest, Any], - job_id) -> Tuple[dict, int, str]: - """ - Get result of job (instance of a process) - - :param request: A request object - :param job_id: ID of job - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - try: - job = self.manager.get_job(job_id) - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'NoSuchJob', job_id - ) - - status = JobStatus[job['status']] - - if status == JobStatus.running: - msg = 'job still running' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'ResultNotReady', msg) - - elif status == JobStatus.accepted: - # NOTE: this case is not mentioned in the specification - msg = 'job accepted but not yet running' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, - request.format, 'ResultNotReady', msg) - - elif status == JobStatus.failed: - msg = 'job failed' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - try: - mimetype, job_output = self.manager.get_job_result(job_id) - except JobResultNotFoundError: - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'JobResultNotFound', job_id - ) - - if mimetype not in (None, FORMAT_TYPES[F_JSON]): - headers['Content-Type'] = mimetype - content = job_output - else: - if request.format == F_JSON: - content = json.dumps(job_output, sort_keys=True, indent=4, - default=json_serial) - else: - # HTML - headers['Content-Type'] = "text/html" - data = { - 'job': {'id': job_id}, - 'result': job_output - } - content = render_j2_template( - self.config, 'jobs/results/index.html', - data, request.locale) - - return headers, HTTPStatus.OK, content - - @pre_process - def delete_job( - self, request: Union[APIRequest, Any], job_id - ) -> Tuple[dict, int, str]: - """ - Delete a process job - - :param job_id: job identifier - - :returns: tuple of headers, status code, content - """ - response_headers = request.get_response_headers( - SYSTEM_LOCALE, **self.api_headers) - try: - success = self.manager.delete_job(job_id) - except JobNotFoundError: - return self.get_exception( - HTTPStatus.NOT_FOUND, response_headers, request.format, - 'NoSuchJob', job_id - ) - else: - if success: - http_status = HTTPStatus.OK - jobs_url = f"{self.base_url}/jobs" - - response = { - 'jobID': job_id, - 'status': JobStatus.dismissed.value, - 'message': 'Job dismissed', - 'progress': 100, - 'links': [{ - 'href': jobs_url, - 'rel': 'up', - 'type': FORMAT_TYPES[F_JSON], - 'title': 'The job list for the current process' - }] - } - else: - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, response_headers, - request.format, 'InternalError', job_id - ) - LOGGER.info(response) - # TODO: this response does not have any headers - return {}, http_status, response - - @gzip - @pre_process - def get_collection_edr_query( - self, request: Union[APIRequest, Any], - dataset, instance, query_type, - location_id=None) -> Tuple[dict, int, str]: - """ - Queries collection EDR - - :param request: APIRequest instance with query params - :param dataset: dataset name - :param instance: instance name - :param query_type: EDR query type - :param location_id: location id of a /location/ query - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(PLUGINS['formatter'].keys()): - return self.get_format_exception(request) - headers = request.get_response_headers(self.default_locale, - **self.api_headers) - collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'collection') - - if dataset not in collections.keys(): - msg = 'Collection not found' - return self.get_exception( - HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) - - LOGGER.debug('Processing query parameters') - - LOGGER.debug('Processing datetime parameter') - datetime_ = request.params.get('datetime') - try: - datetime_ = validate_datetime(collections[dataset]['extents'], - datetime_) - except ValueError as err: - msg = str(err) - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - LOGGER.debug('Processing parameter_names parameter') - parameternames = request.params.get('parameter_names') or [] - if isinstance(parameternames, str): - parameternames = parameternames.split(',') - - bbox = None - if query_type in ['cube', 'locations']: - LOGGER.debug('Processing cube bbox') - try: - bbox = validate_bbox(request.params.get('bbox')) - if not bbox and query_type == 'cube': - raise ValueError('bbox parameter required by cube queries') - except ValueError as err: - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', str(err)) - - LOGGER.debug('Processing coords parameter') - wkt = request.params.get('coords') - - if wkt: - try: - wkt = shapely_loads(wkt) - except WKTReadingError: - msg = 'invalid coords parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - elif query_type not in ['cube', 'locations']: - msg = 'missing coords parameter' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - within = within_units = None - if query_type == 'radius': - LOGGER.debug('Processing within / within-units parameters') - within = request.params.get('within') - within_units = request.params.get('within-units') - - LOGGER.debug('Processing z parameter') - z = request.params.get('z') - - LOGGER.debug('Loading provider') - try: - p = load_plugin('provider', get_provider_by_type( - collections[dataset]['providers'], 'edr')) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if instance is not None and not p.get_instance(instance): - msg = 'Invalid instance identifier' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, - request.format, 'InvalidParameterValue', msg) - - if query_type not in p.get_query_types(): - msg = 'Unsupported query type' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - if parameternames and not any((fld in parameternames) - for fld in p.get_fields().keys()): - msg = 'Invalid parameter_names' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, request.format, - 'InvalidParameterValue', msg) - - query_args = dict( - query_type=query_type, - instance=instance, - format_=request.format, - datetime_=datetime_, - select_properties=parameternames, - wkt=wkt, - z=z, - bbox=bbox, - within=within, - within_units=within_units, - limit=int(self.config['server']['limit']), - location_id=location_id, - ) - - try: - data = p.query(**query_args) - except ProviderGenericError as err: - LOGGER.error(err) - return self.get_exception( - err.http_status_code, headers, request.format, - err.ogc_exception_code, err.message) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'collections/edr/query.html', data, - self.default_locale) - else: - content = to_json(data, self.pretty_print) - - return headers, HTTPStatus.OK, content - - @gzip - @pre_process - @jsonldify - def get_stac_root( - self, request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: - """ - Provide STAC root page - - :param request: APIRequest instance with query params - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - id_ = 'pygeoapi-stac' - stac_version = '1.0.0-rc.2' - stac_url = f'{self.base_url}/stac' - - content = { - 'id': id_, - 'type': 'Catalog', - 'stac_version': stac_version, - 'title': l10n.translate( - self.config['metadata']['identification']['title'], - request.locale), - 'description': l10n.translate( - self.config['metadata']['identification']['description'], - request.locale), - 'links': [] - } - - stac_collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'stac-collection') - - for key, value in stac_collections.items(): - content['links'].append({ - 'rel': 'child', - 'href': f'{stac_url}/{key}?f={F_JSON}', - 'type': FORMAT_TYPES[F_JSON] - }) - content['links'].append({ - 'rel': 'child', - 'href': f'{stac_url}/{key}', - 'type': FORMAT_TYPES[F_HTML] - }) - - if request.format == F_HTML: # render - content = render_j2_template(self.tpl_config, - 'stac/collection.html', - content, request.locale) - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - @gzip - @pre_process - @jsonldify - def get_stac_path(self, request: Union[APIRequest, Any], - path) -> Tuple[dict, int, str]: - """ - Provide STAC resource path - - :param request: APIRequest instance with query params - - :returns: tuple of headers, status code, content - """ - - if not request.is_valid(): - return self.get_format_exception(request) - headers = request.get_response_headers(**self.api_headers) - - dataset = None - LOGGER.debug(f'Path: {path}') - dir_tokens = path.split('/') - if dir_tokens: - dataset = dir_tokens[0] - - stac_collections = filter_dict_by_key_value(self.config['resources'], - 'type', 'stac-collection') - - if dataset not in stac_collections: - msg = 'Collection not found' - return self.get_exception(HTTPStatus.NOT_FOUND, headers, - request.format, 'NotFound', msg) - - LOGGER.debug('Loading provider') - try: - p = load_plugin('provider', get_provider_by_type( - stac_collections[dataset]['providers'], 'stac')) - except ProviderConnectionError as err: - LOGGER.error(err) - msg = 'connection error (check logs)' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'NoApplicableCode', msg) - - id_ = f'{dataset}-stac' - stac_version = '1.0.0-rc.2' - - content = { - 'id': id_, - 'type': 'Catalog', - 'stac_version': stac_version, - 'description': l10n.translate( - stac_collections[dataset]['description'], request.locale), - 'links': [] - } - try: - stac_data = p.get_data_path( - f'{self.base_url}/stac', - path, - path.replace(dataset, '', 1) - ) - except ProviderNotFoundError as err: - LOGGER.error(err) - msg = 'resource not found' - return self.get_exception(HTTPStatus.NOT_FOUND, headers, - request.format, 'NotFound', msg) - except Exception as err: - LOGGER.error(err) - msg = 'data query error' - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, headers, - request.format, 'NoApplicableCode', msg) - - if isinstance(stac_data, dict): - content.update(stac_data) - content['links'].extend( - stac_collections[dataset].get('links', [])) - - if request.format == F_HTML: # render - content['path'] = path - if 'assets' in content: # item view - if content['type'] == 'Collection': - content = render_j2_template( - self.tpl_config, - 'stac/collection_base.html', - content, - request.locale - ) - elif content['type'] == 'Feature': - content = render_j2_template( - self.tpl_config, - 'stac/item.html', - content, - request.locale - ) - else: - msg = f'Unknown STAC type {content.type}' - LOGGER.error(msg) - return self.get_exception( - HTTPStatus.INTERNAL_SERVER_ERROR, - headers, - request.format, - 'NoApplicableCode', - msg) - else: - content = render_j2_template(self.tpl_config, - 'stac/catalog.html', - content, request.locale) - - return headers, HTTPStatus.OK, content - - return headers, HTTPStatus.OK, to_json(content, self.pretty_print) - - else: # send back file - headers.pop('Content-Type', None) - return headers, HTTPStatus.OK, stac_data - - def get_exception(self, status, headers, format_, code, - description) -> Tuple[dict, int, str]: - """ - Exception handler - - :param status: HTTP status code - :param headers: dict of HTTP response headers - :param format_: format string - :param code: OGC API exception code - :param description: OGC API exception code - - :returns: tuple of headers, status, and message - """ - - LOGGER.error(description) - exception = { - 'code': code, - 'type': code, - 'description': description - } - - if format_ == F_HTML: - headers['Content-Type'] = FORMAT_TYPES[F_HTML] - content = render_j2_template( - self.config, 'exception.html', exception, SYSTEM_LOCALE) - else: - content = to_json(exception, self.pretty_print) - - return headers, status, content - - def get_format_exception(self, request) -> Tuple[dict, int, str]: - """ - Returns a format exception. - - :param request: An APIRequest instance. - - :returns: tuple of (headers, status, message) - """ - - # Content-Language is in the system locale (ignore language settings) - headers = request.get_response_headers(SYSTEM_LOCALE, - **self.api_headers) - msg = f'Invalid format: {request.format}' - return self.get_exception( - HTTPStatus.BAD_REQUEST, headers, - request.format, 'InvalidParameterValue', msg) - - def get_collections_url(self): - return f"{self.base_url}/collections" - - @staticmethod - def _create_crs_transform_spec( - config: dict, - query_crs_uri: Optional[str] = None, - ) -> Union[None, CrsTransformSpec]: - """Create a `CrsTransformSpec` instance based on provider config and - *crs* query parameter. - - :param config: Provider config dictionary. - :type config: dict - :param query_crs_uri: Uniform resource identifier of the coordinate - reference system (CRS) specified in query parameter (if specified). - :type query_crs_uri: str, optional - - :raises ValueError: Error raised if the CRS specified in the query - parameter is not in the list of supported CRSs of the provider. - :raises `CRSError`: Error raised if no CRS could be identified from the - query *crs* parameter (URI). - - :returns: `CrsTransformSpec` instance if the CRS specified in query - parameter differs from the storage CRS, else `None`. - :rtype: Union[None, CrsTransformSpec] - """ - # Get storage/default CRS for Collection. - storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) - - if not query_crs_uri: - if storage_crs_uri in DEFAULT_CRS_LIST: - # Could be that storageCRS is - # http://www.opengis.net/def/crs/OGC/1.3/CRS84h - query_crs_uri = storage_crs_uri - else: - query_crs_uri = DEFAULT_CRS - LOGGER.debug(f'no crs parameter, using default: {query_crs_uri}') - - supported_crs_list = get_supported_crs_list(config, DEFAULT_CRS_LIST) - # Check that the crs specified by the query parameter is supported. - if query_crs_uri not in supported_crs_list: - raise ValueError( - f'CRS {query_crs_uri!r} not supported for this ' - 'collection. List of supported CRSs: ' - f'{", ".join(supported_crs_list)}.' - ) - crs_out = get_crs_from_uri(query_crs_uri) - - storage_crs = get_crs_from_uri(storage_crs_uri) - # Check if the crs specified in query parameter differs from the - # storage crs. - if str(storage_crs) != str(crs_out): - LOGGER.debug( - f'CRS transformation: {storage_crs} -> {crs_out}' - ) - return CrsTransformSpec( - source_crs_uri=storage_crs_uri, - source_crs_wkt=storage_crs.to_wkt(), - target_crs_uri=query_crs_uri, - target_crs_wkt=crs_out.to_wkt(), - ) - else: - LOGGER.debug('No CRS transformation') - return None - - @staticmethod - def _set_content_crs_header( - headers: dict, - config: dict, - query_crs_uri: Optional[str] = None, - ): - """Set the *Content-Crs* header in responses from providers of Feature - type. - - :param headers: Response headers dictionary. - :type headers: dict - :param config: Provider config dictionary. - :type config: dict - :param query_crs_uri: Uniform resource identifier of the coordinate - reference system specified in query parameter (if specified). - :type query_crs_uri: str, optional - """ - if query_crs_uri: - content_crs_uri = query_crs_uri - else: - # If empty use default CRS - storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) - if storage_crs_uri in DEFAULT_CRS_LIST: - # Could be that storageCRS is one of the defaults like - # http://www.opengis.net/def/crs/OGC/1.3/CRS84h - content_crs_uri = storage_crs_uri - else: - content_crs_uri = DEFAULT_CRS - - headers['Content-Crs'] = f'<{content_crs_uri}>' - - -def validate_bbox(value=None) -> list: - """ - Helper function to validate bbox parameter - - :param value: `list` of minx, miny, maxx, maxy - - :returns: bbox as `list` of `float` values - """ - - if value is None: - LOGGER.debug('bbox is empty') - return [] - - bbox = value.split(',') - - if len(bbox) not in [4, 6]: - msg = 'bbox should be either 4 values (minx,miny,maxx,maxy) ' \ - 'or 6 values (minx,miny,minz,maxx,maxy,maxz)' - LOGGER.debug(msg) - raise ValueError(msg) - - try: - bbox = [float(c) for c in bbox] - except ValueError as err: - msg = 'bbox values must be numbers' - err.args = (msg,) - LOGGER.debug(msg) - raise - - if (len(bbox) == 4 and bbox[1] > bbox[3]) \ - or (len(bbox) == 6 and bbox[1] > bbox[4]): - msg = 'miny should be less than maxy' - LOGGER.debug(msg) - raise ValueError(msg) - - if (len(bbox) == 4 and bbox[0] > bbox[2]) \ - or (len(bbox) == 6 and bbox[0] > bbox[3]): - msg = 'minx is greater than maxx (possibly antimeridian bbox)' - LOGGER.debug(msg) - - if len(bbox) == 6 and bbox[2] > bbox[5]: - msg = 'minz should be less than maxz' - LOGGER.debug(msg) - raise ValueError(msg) - - return bbox - - -def validate_datetime(resource_def, datetime_=None) -> str: - """ - Helper function to validate temporal parameter - - :param resource_def: `dict` of configuration resource definition - :param datetime_: `str` of datetime parameter - - :returns: `str` of datetime input, if valid - """ - - # TODO: pass datetime to query as a `datetime` object - # we would need to ensure partial dates work accordingly - # as well as setting '..' values to `None` so that underlying - # providers can just assume a `datetime.datetime` object - # - # NOTE: needs testing when passing partials from API to backend - - datetime_invalid = False - - if datetime_ is not None and 'temporal' in resource_def: - - dateparse_begin = partial(dateparse, default=datetime.min) - dateparse_end = partial(dateparse, default=datetime.max) - unix_epoch = datetime(1970, 1, 1, 0, 0, 0) - dateparse_ = partial(dateparse, default=unix_epoch) - - te = resource_def['temporal'] - - try: - if te['begin'] is not None and te['begin'].tzinfo is None: - te['begin'] = te['begin'].replace(tzinfo=pytz.UTC) - if te['end'] is not None and te['end'].tzinfo is None: - te['end'] = te['end'].replace(tzinfo=pytz.UTC) - except AttributeError: - msg = 'Configured times should be RFC3339' - LOGGER.error(msg) - raise ValueError(msg) - - if '/' in datetime_: # envelope - LOGGER.debug('detected time range') - LOGGER.debug('Validating time windows') - - # normalize "" to ".." (actually changes datetime_) - datetime_ = re.sub(r'^/', '../', datetime_) - datetime_ = re.sub(r'/$', '/..', datetime_) - - datetime_begin, datetime_end = datetime_.split('/') - if datetime_begin != '..': - datetime_begin = dateparse_begin(datetime_begin) - if datetime_begin.tzinfo is None: - datetime_begin = datetime_begin.replace( - tzinfo=pytz.UTC) - - if datetime_end != '..': - datetime_end = dateparse_end(datetime_end) - if datetime_end.tzinfo is None: - datetime_end = datetime_end.replace(tzinfo=pytz.UTC) - - datetime_invalid = any([ - (te['end'] is not None and datetime_begin != '..' and - datetime_begin > te['end']), - (te['begin'] is not None and datetime_end != '..' and - datetime_end < te['begin']) - ]) - - else: # time instant - LOGGER.debug('detected time instant') - datetime__ = dateparse_(datetime_) - if datetime__ != '..': - if datetime__.tzinfo is None: - datetime__ = datetime__.replace(tzinfo=pytz.UTC) - datetime_invalid = any([ - (te['begin'] is not None and datetime__ != '..' and - datetime__ < te['begin']), - (te['end'] is not None and datetime__ != '..' and - datetime__ > te['end']) - ]) - - if datetime_invalid: - msg = 'datetime parameter out of range' - LOGGER.debug(msg) - raise ValueError(msg) - - return datetime_ - - -def validate_subset(value: str) -> dict: - """ - Helper function to validate subset parameter - - :param value: `subset` parameter - - :returns: dict of axis/values - """ - - subsets = {} - - for s in value.split(','): - LOGGER.debug(f'Processing subset {s}') - m = re.search(r'(.*)\((.*)\)', s) - subset_name, values = m.group(1, 2) - - if '"' in values: - LOGGER.debug('Values are strings') - if values.count('"') % 2 != 0: - msg = 'Invalid format: subset should be like axis("min"[:"max"])' # noqa - LOGGER.error(msg) - raise ValueError(msg) - try: - LOGGER.debug('Value is an interval') - m = re.search(r'"(\S+)":"(\S+)"', values) - values = list(m.group(1, 2)) - except AttributeError: - LOGGER.debug('Value is point') - m = re.search(r'"(.*)"', values) - values = [m.group(1)] - else: - LOGGER.debug('Values are numbers') - try: - LOGGER.debug('Value is an interval') - m = re.search(r'(\S+):(\S+)', values) - values = list(m.group(1, 2)) - except AttributeError: - LOGGER.debug('Value is point') - values = [values] - - subsets[subset_name] = list(map(get_typed_value, values)) - - return subsets diff --git a/pygeoapi/api/__init__.py b/pygeoapi/api/__init__.py new file mode 100644 index 0000000..0d11c35 --- /dev/null +++ b/pygeoapi/api/__init__.py @@ -0,0 +1,1687 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +""" +Root level code of pygeoapi, parsing content provided by web framework. +Returns content from plugins and sets responses. +""" + +import asyncio +from collections import OrderedDict +from copy import deepcopy +from datetime import datetime +from functools import partial +from gzip import compress +from http import HTTPStatus +import logging +import re +from typing import Any, Tuple, Union, Optional + +from dateutil.parser import parse as dateparse +import pytz + +from pygeoapi import __version__, l10n +from pygeoapi.linked_data import jsonldify, jsonldify_collection +from pygeoapi.log import setup_logger +from pygeoapi.plugin import load_plugin +from pygeoapi.process.manager.base import get_manager +from pygeoapi.provider.base import ( + ProviderConnectionError, ProviderGenericError, ProviderTypeError) + +from pygeoapi.util import ( + CrsTransformSpec, TEMPLATES, UrlPrefetcher, dategetter, + filter_dict_by_key_value, get_api_rules, get_base_url, + get_provider_by_type, get_provider_default, get_typed_value, + get_crs_from_uri, get_supported_crs_list, render_j2_template, to_json +) + +LOGGER = logging.getLogger(__name__) + +#: Return headers for requests (e.g:X-Powered-By) +HEADERS = { + 'Content-Type': 'application/json', + 'X-Powered-By': f'pygeoapi {__version__}' +} + +CHARSET = ['utf-8'] +F_JSON = 'json' +F_HTML = 'html' +F_JSONLD = 'jsonld' +F_GZIP = 'gzip' +F_PNG = 'png' +F_JPEG = 'jpeg' +F_MVT = 'mvt' +F_NETCDF = 'NetCDF' + +#: Formats allowed for ?f= requests (order matters for complex MIME types) +FORMAT_TYPES = OrderedDict(( + (F_HTML, 'text/html'), + (F_JSONLD, 'application/ld+json'), + (F_JSON, 'application/json'), + (F_PNG, 'image/png'), + (F_JPEG, 'image/jpeg'), + (F_MVT, 'application/vnd.mapbox-vector-tile'), + (F_NETCDF, 'application/x-netcdf'), +)) + +#: Locale used for system responses (e.g. exceptions) +SYSTEM_LOCALE = l10n.Locale('en', 'US') + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/landing-page', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/oas30' +] + +OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' + +DEFAULT_CRS_LIST = [ + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', +] + +DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' +DEFAULT_STORAGE_CRS = DEFAULT_CRS + + +def all_apis() -> dict: + """ + Return all supported API modules + + NOTE: this is a function and not a constant to avoid import loops + + :returns: `dict` of API provider type, API module + """ + + from . import (coverages, environmental_data_retrieval, itemtypes, maps, + processes, tiles, stac) + + return { + 'coverage': coverages, + 'edr': environmental_data_retrieval, + 'itemtypes': itemtypes, + 'map': maps, + 'process': processes, + 'tile': tiles, + 'stac': stac + } + + +def pre_process(func): + """ + Decorator that transforms an incoming Request instance specific to the + web framework (i.e. Flask, Starlette or Django) into a generic + :class:`APIRequest` instance. + + :param func: decorated function + + :returns: `func` + """ + + def inner(*args): + cls, req_in = args[:2] + req_out = APIRequest.with_data(req_in, getattr(cls, 'locales', set())) + if len(args) > 2: + return func(cls, req_out, *args[2:]) + else: + return func(cls, req_out) + + return inner + + +# TODO: remove this when all functions have been refactored +def gzip(func): + """ + Decorator that compresses the content of an outgoing API result + instance if the Content-Encoding response header was set to gzip. + + :param func: decorated function + + :returns: `func` + """ + + def inner(*args, **kwargs): + headers, status, content = func(*args, **kwargs) + charset = CHARSET[0] + if F_GZIP in headers.get('Content-Encoding', []): + try: + if isinstance(content, bytes): + # bytes means Content-Type needs to be set upstream + content = compress(content) + else: + headers['Content-Type'] = \ + f"{headers['Content-Type']}; charset={charset}" + content = compress(content.encode(charset)) + except TypeError as err: + headers.pop('Content-Encoding') + LOGGER.error(f'Error in compression: {err}') + + return headers, status, content + + return inner + + +def apply_gzip(headers: dict, content: str | bytes) -> str | bytes: + """ + Compress content if requested in header. + """ + charset = CHARSET[0] + if F_GZIP in headers.get('Content-Encoding', []): + try: + if isinstance(content, bytes): + # bytes means Content-Type needs to be set upstream + content = compress(content) + else: + headers['Content-Type'] = \ + f"{headers['Content-Type']}; charset={charset}" + content = compress(content.encode(charset)) + except TypeError as err: + headers.pop('Content-Encoding') + LOGGER.error(f'Error in compression: {err}') + return content + + +class APIRequest: + """ + Transforms an incoming server-specific Request into an object + with some generic helper methods and properties. + + .. note:: Typically, this instance is created automatically by the + :func:`pre_process` decorator. **Every** API method that has + been routed to a REST endpoint should be decorated by the + :func:`pre_process` function. + Therefore, **all** routed API methods should at least have 1 + argument that holds the (transformed) request. + + The following example API method will: + + - transform the incoming Flask/Starlette/Django `Request` into an + `APIRequest`using the :func:`pre_process` decorator; + - call :meth:`is_valid` to check if the incoming request was valid, i.e. + that the user requested a valid output format or no format at all + (which means the default format); + - call :meth:`API.get_format_exception` if the requested format was + invalid; + - create a `dict` with the appropriate `Content-Type` header for the + requested format and a `Content-Language` header if any specific language + was requested. + + .. code-block:: python + + @pre_process + def example_method(self, request: Union[APIRequest, Any], custom_arg): + if not request.is_valid(): + return self.get_format_exception(request) + + headers = request.get_response_headers() + + # generate response_body here + + return headers, HTTPStatus.OK, response_body + + + The following example API method is similar as the one above, but will also + allow the user to request a non-standard format (e.g. ``f=xml``). + If `xml` was requested, we set the `Content-Type` ourselves. For the + standard formats, the `APIRequest` object sets the `Content-Type`. + + .. code-block:: python + + @pre_process + def example_method(self, request: Union[APIRequest, Any], custom_arg): + if not request.is_valid(['xml']): + return self.get_format_exception(request) + + content_type = 'application/xml' if request.format == 'xml' else None + headers = request.get_response_headers(content_type) + + # generate response_body here + + return headers, HTTPStatus.OK, response_body + + Note that you don't *have* to call :meth:`is_valid`, but that you can also + perform a custom check on the requested output format by looking at the + :attr:`format` property. + Other query parameters are available through the :attr:`params` property as + a `dict`. The request body is available through the :attr:`data` property. + + .. note:: If the request data (body) is important, **always** create a + new `APIRequest` instance using the :meth:`with_data` factory + method. + The :func:`pre_process` decorator will use this automatically. + + :param request: The web platform specific Request instance. + :param supported_locales: List or set of supported Locale instances. + """ + def __init__(self, request, supported_locales): + # Set default request data + self._data = b'' + + # Copy request query parameters + self._args = self._get_params(request) + + # Get path info + if hasattr(request, 'scope'): + self._path_info = request.scope['path'].strip('/') + elif hasattr(request.headers, 'environ'): + self._path_info = request.headers.environ['PATH_INFO'].strip('/') + elif hasattr(request, 'path_info'): + self._path_info = request.path_info + + # Extract locale from params or headers + self._raw_locale, self._locale = self._get_locale(request.headers, + supported_locales) + + # Determine format + self._format = self._get_format(request.headers) + + # Get received headers + self._headers = self.get_request_headers(request.headers) + + # TODO: remove this after all views have been refactored (only used + # in pre_process) + @classmethod + def with_data(cls, request, supported_locales) -> 'APIRequest': + """ + Factory class method to create an `APIRequest` instance with data. + + If the request body is required, an `APIRequest` should always be + instantiated using this class method. The reason for this is, that the + Starlette request body needs to be awaited (async), which cannot be + achieved in the :meth:`__init__` method of the `APIRequest`. + However, `APIRequest` can still be initialized using :meth:`__init__`, + but then the :attr:`data` property value will always be empty. + + :param request: The web platform specific Request instance. + :param supported_locales: List or set of supported Locale instances. + :returns: An `APIRequest` instance with data. + """ + + api_req = cls(request, supported_locales) + if hasattr(request, 'data'): + # Set data from Flask request + api_req._data = request.data + elif hasattr(request, 'body'): + if 'django' in str(request.__class__): + # Set data from Django request + api_req._data = request.body + else: + # Set data from Starlette request after async + # coroutine completion + # TODO: + # this now blocks, but once Flask v2 with async support + # has been implemented, with_data() can become async too + loop = asyncio.get_event_loop() + api_req._data = asyncio.run_coroutine_threadsafe( + request.body(), loop).result(1) + return api_req + + @classmethod + def from_flask(cls, request, supported_locales) -> 'APIRequest': + """Factory class similar to with_data, but only for flask requests""" + api_req = cls(request, supported_locales) + api_req._data = request.data + return api_req + + @classmethod + async def from_starlette(cls, request, supported_locales) -> 'APIRequest': + """Factory class similar to with_data, but only for starlette requests + """ + api_req = cls(request, supported_locales) + api_req._data = await request.body() + return api_req + + @classmethod + def from_django(cls, request, supported_locales) -> 'APIRequest': + """Factory class similar to with_data, but only for django requests""" + api_req = cls(request, supported_locales) + api_req._data = request.body + return api_req + + @staticmethod + def _get_params(request): + """ + Extracts the query parameters from the `Request` object. + + :param request: A Flask or Starlette Request instance + :returns: `ImmutableMultiDict` or empty `dict` + """ + + if hasattr(request, 'args'): + # Return ImmutableMultiDict from Flask request + return request.args + elif hasattr(request, 'query_params'): + # Return ImmutableMultiDict from Starlette request + return request.query_params + elif hasattr(request, 'GET'): + # Return QueryDict from Django GET request + return request.GET + elif hasattr(request, 'POST'): + # Return QueryDict from Django GET request + return request.POST + LOGGER.debug('No query parameters found') + return {} + + def _get_locale(self, headers, supported_locales): + """ + Detects locale from "lang=" param or `Accept-Language` + header. Returns a tuple of (raw, locale) if found in params or headers. + Returns a tuple of (raw default, default locale) if not found. + + :param headers: A dict with Request headers + :param supported_locales: List or set of supported Locale instances + :returns: A tuple of (str, Locale) + """ + + raw = None + try: + default_locale = l10n.str2locale(supported_locales[0]) + except (TypeError, IndexError, l10n.LocaleError) as err: + # This should normally not happen, since the API class already + # loads the supported languages from the config, which raises + # a LocaleError if any of these languages are invalid. + LOGGER.error(err) + raise ValueError(f"{self.__class__.__name__} must be initialized" + f"with a list of valid supported locales") + + for func, mapping in ((l10n.locale_from_params, self._args), + (l10n.locale_from_headers, headers)): + loc_str = func(mapping) + if loc_str: + if not raw: + # This is the first-found locale string: set as raw + raw = loc_str + # Check if locale string is a good match for the UI + loc = l10n.best_match(loc_str, supported_locales) + is_override = func is l10n.locale_from_params + if loc != default_locale or is_override: + return raw, loc + + return raw, default_locale + + def _get_format(self, headers) -> Union[str, None]: + """ + Get `Request` format type from query parameters or headers. + + :param headers: Dict of Request headers + :returns: format value or None if not found/specified + """ + + # Optional f=html or f=json query param + # Overrides Accept header and might differ from FORMAT_TYPES + format_ = (self._args.get('f') or '').strip() + if format_: + return format_ + + # Format not specified: get from Accept headers (MIME types) + # e.g. format_ = 'text/html' + h = headers.get('accept', headers.get('Accept', '')).strip() # noqa + (fmts, mimes) = zip(*FORMAT_TYPES.items()) + # basic support for complex types (i.e. with "q=0.x") + for type_ in (t.split(';')[0].strip() for t in h.split(',') if t): + if type_ in mimes: + idx_ = mimes.index(type_) + format_ = fmts[idx_] + break + + return format_ or None + + @property + def data(self) -> bytes: + """Returns the additional data send with the Request (bytes)""" + return self._data + + @property + def params(self) -> dict: + """Returns the Request query parameters dict""" + return self._args + + @property + def path_info(self) -> str: + """Returns the web server request path info part""" + return self._path_info + + @property + def locale(self) -> l10n.Locale: + """ + Returns the user-defined locale from the request object. + If no locale has been defined or if it is invalid, + the default server locale is returned. + + .. note:: The locale here determines the language in which pygeoapi + should return its responses. This may not be the language + that the user requested. It may also not be the language + that is supported by a collection provider, for example. + For this reason, you should pass the `raw_locale` property + to the :func:`l10n.get_plugin_locale` function, so that + the best match for the provider can be determined. + + :returns: babel.core.Locale + """ + + return self._locale + + @property + def raw_locale(self) -> Union[str, None]: + """ + Returns the raw locale string from the `Request` object. + If no "lang" query parameter or `Accept-Language` header was found, + `None` is returned. + Pass this value to the :func:`l10n.get_plugin_locale` function to let + the provider determine a best match for the locale, which may be + different from the locale used by pygeoapi's UI. + + :returns: a locale string or None + """ + + return self._raw_locale + + @property + def format(self) -> Union[str, None]: + """ + Returns the content type format from the + request query parameters or headers. + + :returns: Format name or None + """ + + return self._format + + @property + def headers(self) -> dict: + """ + Returns the dictionary of the headers from + the request. + + :returns: Request headers dictionary + """ + + return self._headers + + def get_linkrel(self, format_: str) -> str: + """ + Returns the hyperlink relationship (rel) attribute value for + the given API format string. + + The string is compared against the request format and if it matches, + the value 'self' is returned. Otherwise, 'alternate' is returned. + However, if `format_` is 'json' and *no* request format was found, + the relationship 'self' is returned as well (JSON is the default). + + :param format_: The format to compare the request format against. + :returns: A string 'self' or 'alternate'. + """ + + fmt = format_.lower() + if fmt == self._format or (fmt == F_JSON and not self._format): + return 'self' + return 'alternate' + + def is_valid(self, additional_formats=None) -> bool: + """ + Returns True if: + - the format is not set (None) + - the requested format is supported + - the requested format exists in a list if additional formats + + .. note:: Format names are matched in a case-insensitive manner. + + :param additional_formats: Optional additional supported formats list + + :returns: bool + """ + + if not self._format: + return True + if self._format in FORMAT_TYPES.keys(): + return True + if self._format in (f.lower() for f in (additional_formats or ())): + return True + return False + + def get_response_headers(self, force_lang: l10n.Locale = None, + force_type: str = None, + force_encoding: str = None, + **custom_headers) -> dict: + """ + Prepares and returns a dictionary with Response object headers. + + This method always adds a 'Content-Language' header, where the value + is determined by the 'lang' query parameter or 'Accept-Language' + header from the request. + If no language was requested, the default pygeoapi language is used, + unless a `force_lang` override was specified (see notes below). + + A 'Content-Type' header is also always added to the response. + If the user does not specify `force_type`, the header is based on + the `format` APIRequest property. If that is invalid, the default MIME + type `application/json` is used. + + ..note:: If a `force_lang` override is applied, that language + is always set as the 'Content-Language', regardless of + a 'lang' query parameter or 'Accept-Language' header. + If an API response always needs to be in the same + language, 'force_lang' should be set to that language. + + :param force_lang: An optional Content-Language header override. + :param force_type: An optional Content-Type header override. + :param force_encoding: An optional Content-Encoding header override. + :returns: A header dict + """ + + headers = HEADERS.copy() + headers.update(**custom_headers) + l10n.set_response_language(headers, force_lang or self._locale) + if force_type: + # Set custom MIME type if specified + headers['Content-Type'] = force_type + elif self.is_valid() and self._format: + # Set MIME type for valid formats + headers['Content-Type'] = FORMAT_TYPES[self._format] + + if F_GZIP in FORMAT_TYPES: + if force_encoding: + headers['Content-Encoding'] = force_encoding + elif F_GZIP in self._headers.get('Accept-Encoding', ''): + headers['Content-Encoding'] = F_GZIP + + return headers + + def get_request_headers(self, headers) -> dict: + """ + Obtains and returns a dictionary with Request object headers. + + This method adds the headers of the original request and + makes them available to the API object. + + :returns: A header dict + """ + + headers_ = {item[0]: item[1] for item in headers.items()} + return headers_ + + +class API: + """API object""" + + def __init__(self, config, openapi): + """ + constructor + + :param config: configuration dict + :param openapi: openapi dict + + :returns: `pygeoapi.API` instance + """ + + self.config = config + self.openapi = openapi + self.api_headers = get_api_rules(self.config).response_headers + self.base_url = get_base_url(self.config) + self.prefetcher = UrlPrefetcher() + + CHARSET[0] = config['server'].get('encoding', 'utf-8') + if config['server'].get('gzip'): + FORMAT_TYPES[F_GZIP] = 'application/gzip' + FORMAT_TYPES.move_to_end(F_JSON) + + # Process language settings (first locale is default!) + self.locales = l10n.get_locales(config) + self.default_locale = self.locales[0] + + if 'templates' not in self.config['server']: + self.config['server']['templates'] = {'path': TEMPLATES} + + if 'pretty_print' not in self.config['server']: + self.config['server']['pretty_print'] = False + + self.pretty_print = self.config['server']['pretty_print'] + + setup_logger(self.config['logging']) + + # Create config clone for HTML templating with modified base URL + self.tpl_config = deepcopy(self.config) + self.tpl_config['server']['url'] = self.base_url + + self.manager = get_manager(self.config) + LOGGER.info('Process manager plugin loaded') + + @gzip + @pre_process + @jsonldify + def landing_page(self, + request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: + """ + Provide API landing page + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + fcm = { + 'links': [], + 'title': l10n.translate( + self.config['metadata']['identification']['title'], + request.locale), + 'description': + l10n.translate( + self.config['metadata']['identification']['description'], + request.locale) + } + + LOGGER.debug('Creating links') + # TODO: put title text in config or translatable files? + fcm['links'] = [{ + 'rel': request.get_linkrel(F_JSON), + 'type': FORMAT_TYPES[F_JSON], + 'title': 'This document as JSON', + 'href': f"{self.base_url}?f={F_JSON}" + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': 'This document as RDF (JSON-LD)', + 'href': f"{self.base_url}?f={F_JSONLD}" + }, { + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': 'This document as HTML', + 'href': f"{self.base_url}?f={F_HTML}", + 'hreflang': self.default_locale + }, { + 'rel': 'service-desc', + 'type': 'application/vnd.oai.openapi+json;version=3.0', + 'title': 'The OpenAPI definition as JSON', + 'href': f"{self.base_url}/openapi" + }, { + 'rel': 'service-doc', + 'type': FORMAT_TYPES[F_HTML], + 'title': 'The OpenAPI definition as HTML', + 'href': f"{self.base_url}/openapi?f={F_HTML}", + 'hreflang': self.default_locale + }, { + 'rel': 'conformance', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'Conformance', + 'href': f"{self.base_url}/conformance" + }, { + 'rel': 'data', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'Collections', + 'href': self.get_collections_url() + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/processes', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'Processes', + 'href': f"{self.base_url}/processes" + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'Jobs', + 'href': f"{self.base_url}/jobs" + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'The list of supported tiling schemes (as JSON)', + 'href': f"{self.base_url}/TileMatrixSets?f=json" + }, { + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes', + 'type': FORMAT_TYPES[F_HTML], + 'title': 'The list of supported tiling schemes (as HTML)', + 'href': f"{self.base_url}/TileMatrixSets?f=html" + }] + + headers = request.get_response_headers(**self.api_headers) + if request.format == F_HTML: # render + + fcm['processes'] = False + fcm['stac'] = False + fcm['collection'] = False + + if filter_dict_by_key_value(self.config['resources'], + 'type', 'process'): + fcm['processes'] = True + + if filter_dict_by_key_value(self.config['resources'], + 'type', 'stac-collection'): + fcm['stac'] = True + + if filter_dict_by_key_value(self.config['resources'], + 'type', 'collection'): + fcm['collection'] = True + + content = render_j2_template(self.tpl_config, 'landing_page.html', + fcm, request.locale) + return headers, HTTPStatus.OK, content + + if request.format == F_JSONLD: + return headers, HTTPStatus.OK, to_json( + self.fcmld, self.pretty_print) + + return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) + + @gzip + @pre_process + def openapi_(self, request: Union[APIRequest, Any]) -> Tuple[ + dict, int, str]: + """ + Provide OpenAPI document + + :param request: A request object + :param openapi: dict of OpenAPI definition + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + headers = request.get_response_headers(**self.api_headers) + + if request.format == F_HTML: + template = 'openapi/swagger.html' + if request._args.get('ui') == 'redoc': + template = 'openapi/redoc.html' + + path = f'{self.base_url}/openapi' + data = { + 'openapi-document-path': path + } + content = render_j2_template(self.tpl_config, template, data, + request.locale) + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/vnd.oai.openapi+json;version=3.0' # noqa + + if isinstance(self.openapi, dict): + return headers, HTTPStatus.OK, to_json(self.openapi, + self.pretty_print) + else: + return headers, HTTPStatus.OK, self.openapi + + @gzip + @pre_process + def conformance(self, + request: Union[APIRequest, Any]) -> Tuple[dict, int, str]: + """ + Provide conformance definition + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + apis_dict = all_apis() + + if not request.is_valid(): + return self.get_format_exception(request) + + conformance_list = CONFORMANCE_CLASSES + + for key, value in self.config['resources'].items(): + if value['type'] == 'process': + conformance_list.extend( + apis_dict['process'].CONFORMANCE_CLASSES) + else: + for provider in value['providers']: + if provider['type'] in apis_dict: + conformance_list.extend( + apis_dict[provider['type']].CONFORMANCE_CLASSES) + if provider['type'] == 'feature': + conformance_list.extend( + apis_dict['itemtypes'].CONFORMANCE_CLASSES_FEATURES) # noqa + if provider['type'] == 'record': + conformance_list.extend( + apis_dict['itemtypes'].CONFORMANCE_CLASSES_RECORDS) + + conformance = { + 'conformsTo': sorted(list(set(conformance_list))) + } + + headers = request.get_response_headers(**self.api_headers) + if request.format == F_HTML: # render + content = render_j2_template(self.tpl_config, 'conformance.html', + conformance, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(conformance, self.pretty_print) + + @gzip + @pre_process + @jsonldify + def describe_collections(self, request: Union[APIRequest, Any], + dataset=None) -> Tuple[dict, int, str]: + """ + Provide collection metadata + + :param request: A request object + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + headers = request.get_response_headers(**self.api_headers) + + fcm = { + 'collections': [], + 'links': [] + } + + collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'collection') + + if all([dataset is not None, dataset not in collections.keys()]): + msg = 'Collection not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + if dataset is not None: + collections_dict = { + k: v for k, v in collections.items() if k == dataset + } + else: + collections_dict = collections + + LOGGER.debug('Creating collections') + for k, v in collections_dict.items(): + if v.get('visibility', 'default') == 'hidden': + LOGGER.debug(f'Skipping hidden layer: {k}') + continue + collection_data = get_provider_default(v['providers']) + collection_data_type = collection_data['type'] + + collection_data_format = None + + if 'format' in collection_data: + collection_data_format = collection_data['format'] + + is_vector_tile = (collection_data_type == 'tile' and + collection_data_format['name'] not + in [F_PNG, F_JPEG]) + + collection = { + 'id': k, + 'title': l10n.translate(v['title'], request.locale), + 'description': l10n.translate(v['description'], request.locale), # noqa + 'keywords': l10n.translate(v['keywords'], request.locale), + 'links': [] + } + + bbox = v['extents']['spatial']['bbox'] + # The output should be an array of bbox, so if the user only + # provided a single bbox, wrap it in a array. + if not isinstance(bbox[0], list): + bbox = [bbox] + collection['extent'] = { + 'spatial': { + 'bbox': bbox + } + } + if 'crs' in v['extents']['spatial']: + collection['extent']['spatial']['crs'] = \ + v['extents']['spatial']['crs'] + + t_ext = v.get('extents', {}).get('temporal', {}) + if t_ext: + begins = dategetter('begin', t_ext) + ends = dategetter('end', t_ext) + collection['extent']['temporal'] = { + 'interval': [[begins, ends]] + } + if 'trs' in t_ext: + collection['extent']['temporal']['trs'] = t_ext['trs'] + + LOGGER.debug('Processing configured collection links') + for link in l10n.translate(v.get('links', []), request.locale): + lnk = { + 'type': link['type'], + 'rel': link['rel'], + 'title': l10n.translate(link['title'], request.locale), + 'href': l10n.translate(link['href'], request.locale), + } + if 'hreflang' in link: + lnk['hreflang'] = l10n.translate( + link['hreflang'], request.locale) + content_length = link.get('length', 0) + + if lnk['rel'] == 'enclosure' and content_length == 0: + # Issue HEAD request for enclosure links without length + lnk_headers = self.prefetcher.get_headers(lnk['href']) + content_length = int(lnk_headers.get('content-length', 0)) + content_type = lnk_headers.get('content-type', lnk['type']) + if content_length == 0: + # Skip this (broken) link + LOGGER.debug(f"Enclosure {lnk['href']} is invalid") + continue + if content_type != lnk['type']: + # Update content type if different from specified + lnk['type'] = content_type + LOGGER.debug( + f"Fixed media type for enclosure {lnk['href']}") + + if content_length > 0: + lnk['length'] = content_length + + collection['links'].append(lnk) + + # TODO: provide translations + LOGGER.debug('Adding JSON and HTML link relations') + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'root', + 'title': 'The landing page of this server as JSON', + 'href': f"{self.base_url}?f={F_JSON}" + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'root', + 'title': 'The landing page of this server as HTML', + 'href': f"{self.base_url}?f={F_HTML}" + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{self.get_collections_url()}/{k}?f={F_JSON}' + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{self.get_collections_url()}/{k}?f={F_JSONLD}' + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{self.get_collections_url()}/{k}?f={F_HTML}' + }) + + if collection_data_type in ['feature', 'coverage', 'record']: + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': f'{OGC_RELTYPES_BASE}/schema', + 'title': 'Schema of collection in JSON', + 'href': f'{self.get_collections_url()}/{k}/schema?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': f'{OGC_RELTYPES_BASE}/schema', + 'title': 'Schema of collection in HTML', + 'href': f'{self.get_collections_url()}/{k}/schema?f={F_HTML}' # noqa + }) + + if is_vector_tile or collection_data_type in ['feature', 'record']: + # TODO: translate + collection['itemType'] = collection_data_type + LOGGER.debug('Adding feature/record based links') + collection['links'].append({ + 'type': 'application/schema+json', + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', + 'title': 'Queryables for this collection as JSON', + 'href': f'{self.get_collections_url()}/{k}/queryables?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/queryables', + 'title': 'Queryables for this collection as HTML', + 'href': f'{self.get_collections_url()}/{k}/queryables?f={F_HTML}' # noqa + }) + collection['links'].append({ + 'type': 'application/geo+json', + 'rel': 'items', + 'title': 'items as GeoJSON', + 'href': f'{self.get_collections_url()}/{k}/items?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': 'items', + 'title': 'items as RDF (GeoJSON-LD)', + 'href': f'{self.get_collections_url()}/{k}/items?f={F_JSONLD}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'items', + 'title': 'Items as HTML', + 'href': f'{self.get_collections_url()}/{k}/items?f={F_HTML}' # noqa + }) + + # OAPIF Part 2 - list supported CRSs and StorageCRS + if collection_data_type == 'feature': + collection['crs'] = get_supported_crs_list(collection_data, DEFAULT_CRS_LIST) # noqa + collection['storageCRS'] = collection_data.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa + if 'storage_crs_coordinate_epoch' in collection_data: + collection['storageCrsCoordinateEpoch'] = collection_data.get('storage_crs_coordinate_epoch') # noqa + + elif collection_data_type == 'coverage': + # TODO: translate + LOGGER.debug('Adding coverage based links') + collection['links'].append({ + 'type': 'application/prs.coverage+json', + 'rel': f'{OGC_RELTYPES_BASE}/coverage', + 'title': 'Coverage data', + 'href': f'{self.get_collections_url()}/{k}/coverage?f={F_JSON}' # noqa + }) + if collection_data_format is not None: + collection['links'].append({ + 'type': collection_data_format['mimetype'], + 'rel': f'{OGC_RELTYPES_BASE}/coverage', + 'title': f"Coverage data as {collection_data_format['name']}", # noqa + 'href': f"{self.get_collections_url()}/{k}/coverage?f={collection_data_format['name']}" # noqa + }) + if dataset is not None: + LOGGER.debug('Creating extended coverage metadata') + try: + provider_def = get_provider_by_type( + self.config['resources'][k]['providers'], + 'coverage') + p = load_plugin('provider', provider_def) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, + headers, request.format, + 'NoApplicableCode', msg) + except ProviderTypeError: + pass + else: + collection['extent']['spatial']['grid'] = [{ + 'cellsCount': p._coverage_properties['width'], + 'resolution': p._coverage_properties['resx'] + }, { + 'cellsCount': p._coverage_properties['height'], + 'resolution': p._coverage_properties['resy'] + }] + + try: + tile = get_provider_by_type(v['providers'], 'tile') + p = load_plugin('provider', tile) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, + headers, request.format, + 'NoApplicableCode', msg) + except ProviderTypeError: + tile = None + + if tile: + # TODO: translate + + LOGGER.debug('Adding tile links') + collection['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa + 'title': 'Tiles as JSON', + 'href': f'{self.get_collections_url()}/{k}/tiles?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': f'http://www.opengis.net/def/rel/ogc/1.0/tilesets-{p.tile_type}', # noqa + 'title': 'Tiles as HTML', + 'href': f'{self.get_collections_url()}/{k}/tiles?f={F_HTML}' # noqa + }) + + try: + map_ = get_provider_by_type(v['providers'], 'map') + except ProviderTypeError: + map_ = None + + if map_: + LOGGER.debug('Adding map links') + + map_mimetype = map_['format']['mimetype'] + map_format = map_['format']['name'] + + collection['links'].append({ + 'type': map_mimetype, + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/map', + 'title': f'Map as {map_format}', + 'href': f"{self.get_collections_url()}/{k}/map?f={map_format}" # noqa + }) + + try: + edr = get_provider_by_type(v['providers'], 'edr') + p = load_plugin('provider', edr) + except ProviderConnectionError: + msg = 'connection error (check logs)' + return self.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'NoApplicableCode', msg) + except ProviderTypeError: + edr = None + + if edr: + # TODO: translate + LOGGER.debug('Adding EDR links') + parameters = p.get_fields() + if parameters: + collection['parameter_names'] = {} + for key, value in parameters.items(): + collection['parameter_names'][key] = { + 'id': key, + 'type': 'Parameter', + 'name': value['title'], + 'unit': { + 'label': { + 'en': value['title'] + }, + 'symbol': { + 'value': value['x-ogc-unit'], + 'type': 'http://www.opengis.net/def/uom/UCUM/' # noqa + } + } + } + + for qt in p.get_query_types(): + collection['links'].append({ + 'type': 'application/json', + 'rel': 'data', + 'title': f'{qt} query for this collection as JSON', + 'href': f'{self.get_collections_url()}/{k}/{qt}?f={F_JSON}' # noqa + }) + collection['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'data', + 'title': f'{qt} query for this collection as HTML', + 'href': f'{self.get_collections_url()}/{k}/{qt}?f={F_HTML}' # noqa + }) + + if dataset is not None and k == dataset: + fcm = collection + break + + fcm['collections'].append(collection) + + if dataset is None: + # TODO: translate + fcm['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{self.get_collections_url()}?f={F_JSON}' + }) + fcm['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{self.get_collections_url()}?f={F_JSONLD}' + }) + fcm['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{self.get_collections_url()}?f={F_HTML}' + }) + + if request.format == F_HTML: # render + fcm['collections_path'] = self.get_collections_url() + if dataset is not None: + content = render_j2_template(self.tpl_config, + 'collections/collection.html', + fcm, request.locale) + else: + content = render_j2_template(self.tpl_config, + 'collections/index.html', fcm, + request.locale) + + return headers, HTTPStatus.OK, content + + if request.format == F_JSONLD: + jsonld = self.fcmld.copy() + if dataset is not None: + jsonld['dataset'] = jsonldify_collection(self, fcm, + request.locale) + else: + jsonld['dataset'] = [ + jsonldify_collection(self, c, request.locale) + for c in fcm.get('collections', []) + ] + return headers, HTTPStatus.OK, to_json(jsonld, self.pretty_print) + + return headers, HTTPStatus.OK, to_json(fcm, self.pretty_print) + + @gzip + @pre_process + def get_collection_schema(self, request: Union[APIRequest, Any], + dataset) -> Tuple[dict, int, str]: + """ + Returns a collection schema + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**self.api_headers) + + if any([dataset is None, + dataset not in self.config['resources'].keys()]): + + msg = 'Collection not found' + return self.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection schema') + try: + LOGGER.debug('Loading feature provider') + p = load_plugin('provider', get_provider_by_type( + self.config['resources'][dataset]['providers'], 'feature')) + except ProviderTypeError: + try: + LOGGER.debug('Loading coverage provider') + p = load_plugin('provider', get_provider_by_type( + self.config['resources'][dataset]['providers'], 'coverage')) # noqa + except ProviderTypeError: + LOGGER.debug('Loading record provider') + p = load_plugin('provider', get_provider_by_type( + self.config['resources'][dataset]['providers'], 'record')) + except ProviderGenericError as err: + LOGGER.error(err) + return self.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + schema = { + 'type': 'object', + 'title': l10n.translate( + self.config['resources'][dataset]['title'], request.locale), + 'properties': {}, + '$schema': 'http://json-schema.org/draft/2019-09/schema', + '$id': f'{self.get_collections_url()}/{dataset}/schema' + } + + if p.type != 'coverage': + schema['properties']['geometry'] = { + '$ref': 'https://geojson.org/schema/Geometry.json', + 'x-ogc-role': 'primary-geometry' + } + + for k, v in p.fields.items(): + schema['properties'][k] = v + + if k == p.id_field: + schema['properties'][k]['x-ogc-role'] = 'id' + if k == p.time_field: + schema['properties'][k]['x-ogc-role'] = 'primary-instant' + + if request.format == F_HTML: # render + schema['title'] = l10n.translate( + self.config['resources'][dataset]['title'], request.locale) + + schema['collections_path'] = self.get_collections_url() + + content = render_j2_template(self.tpl_config, + 'collections/schema.html', + schema, request.locale) + + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/schema+json' + + return headers, HTTPStatus.OK, to_json(schema, self.pretty_print) + + def get_exception(self, status, headers, format_, code, + description) -> Tuple[dict, int, str]: + """ + Exception handler + + :param status: HTTP status code + :param headers: dict of HTTP response headers + :param format_: format string + :param code: OGC API exception code + :param description: OGC API exception code + + :returns: tuple of headers, status, and message + """ + + LOGGER.error(description) + exception = { + 'code': code, + 'type': code, + 'description': description + } + + if format_ == F_HTML: + headers['Content-Type'] = FORMAT_TYPES[F_HTML] + content = render_j2_template( + self.config, 'exception.html', exception, SYSTEM_LOCALE) + else: + content = to_json(exception, self.pretty_print) + + return headers, status, content + + def get_format_exception(self, request) -> Tuple[dict, int, str]: + """ + Returns a format exception. + + :param request: An APIRequest instance. + + :returns: tuple of (headers, status, message) + """ + + # Content-Language is in the system locale (ignore language settings) + headers = request.get_response_headers(SYSTEM_LOCALE, + **self.api_headers) + msg = f'Invalid format: {request.format}' + return self.get_exception( + HTTPStatus.BAD_REQUEST, headers, + request.format, 'InvalidParameterValue', msg) + + def get_collections_url(self): + return f"{self.base_url}/collections" + + @staticmethod + def _create_crs_transform_spec( + config: dict, + query_crs_uri: Optional[str] = None, + ) -> Union[None, CrsTransformSpec]: + """Create a `CrsTransformSpec` instance based on provider config and + *crs* query parameter. + + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system (CRS) specified in query parameter (if specified). + :type query_crs_uri: str, optional + + :raises ValueError: Error raised if the CRS specified in the query + parameter is not in the list of supported CRSs of the provider. + :raises `CRSError`: Error raised if no CRS could be identified from the + query *crs* parameter (URI). + + :returns: `CrsTransformSpec` instance if the CRS specified in query + parameter differs from the storage CRS, else `None`. + :rtype: Union[None, CrsTransformSpec] + """ + # Get storage/default CRS for Collection. + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + + if not query_crs_uri: + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + query_crs_uri = storage_crs_uri + else: + query_crs_uri = DEFAULT_CRS + LOGGER.debug(f'no crs parameter, using default: {query_crs_uri}') + + supported_crs_list = get_supported_crs_list(config, DEFAULT_CRS_LIST) + # Check that the crs specified by the query parameter is supported. + if query_crs_uri not in supported_crs_list: + raise ValueError( + f'CRS {query_crs_uri!r} not supported for this ' + 'collection. List of supported CRSs: ' + f'{", ".join(supported_crs_list)}.' + ) + crs_out = get_crs_from_uri(query_crs_uri) + + storage_crs = get_crs_from_uri(storage_crs_uri) + # Check if the crs specified in query parameter differs from the + # storage crs. + if str(storage_crs) != str(crs_out): + LOGGER.debug( + f'CRS transformation: {storage_crs} -> {crs_out}' + ) + return CrsTransformSpec( + source_crs_uri=storage_crs_uri, + source_crs_wkt=storage_crs.to_wkt(), + target_crs_uri=query_crs_uri, + target_crs_wkt=crs_out.to_wkt(), + ) + else: + LOGGER.debug('No CRS transformation') + return None + + @staticmethod + def _set_content_crs_header( + headers: dict, + config: dict, + query_crs_uri: Optional[str] = None, + ): + """Set the *Content-Crs* header in responses from providers of Feature + type. + + :param headers: Response headers dictionary. + :type headers: dict + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system specified in query parameter (if specified). + :type query_crs_uri: str, optional + """ + if query_crs_uri: + content_crs_uri = query_crs_uri + else: + # If empty use default CRS + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is one of the defaults like + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + content_crs_uri = storage_crs_uri + else: + content_crs_uri = DEFAULT_CRS + + headers['Content-Crs'] = f'<{content_crs_uri}>' + + +def validate_bbox(value=None) -> list: + """ + Helper function to validate bbox parameter + + :param value: `list` of minx, miny, maxx, maxy + + :returns: bbox as `list` of `float` values + """ + + if value is None: + LOGGER.debug('bbox is empty') + return [] + + bbox = value.split(',') + + if len(bbox) not in [4, 6]: + msg = 'bbox should be either 4 values (minx,miny,maxx,maxy) ' \ + 'or 6 values (minx,miny,minz,maxx,maxy,maxz)' + LOGGER.debug(msg) + raise ValueError(msg) + + try: + bbox = [float(c) for c in bbox] + except ValueError as err: + msg = 'bbox values must be numbers' + err.args = (msg,) + LOGGER.debug(msg) + raise + + if (len(bbox) == 4 and bbox[1] > bbox[3]) \ + or (len(bbox) == 6 and bbox[1] > bbox[4]): + msg = 'miny should be less than maxy' + LOGGER.debug(msg) + raise ValueError(msg) + + if (len(bbox) == 4 and bbox[0] > bbox[2]) \ + or (len(bbox) == 6 and bbox[0] > bbox[3]): + msg = 'minx is greater than maxx (possibly antimeridian bbox)' + LOGGER.debug(msg) + + if len(bbox) == 6 and bbox[2] > bbox[5]: + msg = 'minz should be less than maxz' + LOGGER.debug(msg) + raise ValueError(msg) + + return bbox + + +def validate_datetime(resource_def, datetime_=None) -> str: + """ + Helper function to validate temporal parameter + + :param resource_def: `dict` of configuration resource definition + :param datetime_: `str` of datetime parameter + + :returns: `str` of datetime input, if valid + """ + + # TODO: pass datetime to query as a `datetime` object + # we would need to ensure partial dates work accordingly + # as well as setting '..' values to `None` so that underlying + # providers can just assume a `datetime.datetime` object + # + # NOTE: needs testing when passing partials from API to backend + + datetime_invalid = False + + if datetime_ is not None and 'temporal' in resource_def: + + dateparse_begin = partial(dateparse, default=datetime.min) + dateparse_end = partial(dateparse, default=datetime.max) + unix_epoch = datetime(1970, 1, 1, 0, 0, 0) + dateparse_ = partial(dateparse, default=unix_epoch) + + te = resource_def['temporal'] + + try: + if te['begin'] is not None and te['begin'].tzinfo is None: + te['begin'] = te['begin'].replace(tzinfo=pytz.UTC) + if te['end'] is not None and te['end'].tzinfo is None: + te['end'] = te['end'].replace(tzinfo=pytz.UTC) + except AttributeError: + msg = 'Configured times should be RFC3339' + LOGGER.error(msg) + raise ValueError(msg) + + if '/' in datetime_: # envelope + LOGGER.debug('detected time range') + LOGGER.debug('Validating time windows') + + # normalize "" to ".." (actually changes datetime_) + datetime_ = re.sub(r'^/', '../', datetime_) + datetime_ = re.sub(r'/$', '/..', datetime_) + + datetime_begin, datetime_end = datetime_.split('/') + if datetime_begin != '..': + datetime_begin = dateparse_begin(datetime_begin) + if datetime_begin.tzinfo is None: + datetime_begin = datetime_begin.replace( + tzinfo=pytz.UTC) + + if datetime_end != '..': + datetime_end = dateparse_end(datetime_end) + if datetime_end.tzinfo is None: + datetime_end = datetime_end.replace(tzinfo=pytz.UTC) + + datetime_invalid = any([ + (te['end'] is not None and datetime_begin != '..' and + datetime_begin > te['end']), + (te['begin'] is not None and datetime_end != '..' and + datetime_end < te['begin']) + ]) + + else: # time instant + LOGGER.debug('detected time instant') + datetime__ = dateparse_(datetime_) + if datetime__ != '..': + if datetime__.tzinfo is None: + datetime__ = datetime__.replace(tzinfo=pytz.UTC) + datetime_invalid = any([ + (te['begin'] is not None and datetime__ != '..' and + datetime__ < te['begin']), + (te['end'] is not None and datetime__ != '..' and + datetime__ > te['end']) + ]) + + if datetime_invalid: + msg = 'datetime parameter out of range' + LOGGER.debug(msg) + raise ValueError(msg) + + return datetime_ + + +def validate_subset(value: str) -> dict: + """ + Helper function to validate subset parameter + + :param value: `subset` parameter + + :returns: dict of axis/values + """ + + subsets = {} + + for s in value.split(','): + LOGGER.debug(f'Processing subset {s}') + m = re.search(r'(.*)\((.*)\)', s) + subset_name, values = m.group(1, 2) + + if '"' in values: + LOGGER.debug('Values are strings') + if values.count('"') % 2 != 0: + msg = 'Invalid format: subset should be like axis("min"[:"max"])' # noqa + LOGGER.error(msg) + raise ValueError(msg) + try: + LOGGER.debug('Value is an interval') + m = re.search(r'"(\S+)":"(\S+)"', values) + values = list(m.group(1, 2)) + except AttributeError: + LOGGER.debug('Value is point') + m = re.search(r'"(.*)"', values) + values = [m.group(1)] + else: + LOGGER.debug('Values are numbers') + try: + LOGGER.debug('Value is an interval') + m = re.search(r'(\S+):(\S+)', values) + values = list(m.group(1, 2)) + except AttributeError: + LOGGER.debug('Value is point') + values = [values] + + subsets[subset_name] = list(map(get_typed_value, values)) + + return subsets diff --git a/pygeoapi/api/coverages.py b/pygeoapi/api/coverages.py new file mode 100644 index 0000000..01e5638 --- /dev/null +++ b/pygeoapi/api/coverages.py @@ -0,0 +1,253 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import logging +from http import HTTPStatus +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.plugin import load_plugin +from pygeoapi.provider.base import ProviderGenericError, ProviderTypeError +from pygeoapi.util import ( + filter_dict_by_key_value, get_provider_by_type, to_json +) + +from . import ( + APIRequest, API, F_JSON, SYSTEM_LOCALE, validate_bbox, validate_datetime, + validate_subset +) + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/geodata-coverage', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-subset', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-rangesubset', # noqa + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-bbox', + 'http://www.opengis.net/spec/ogcapi-coverages-1/1.0/conf/coverage-datetime' +] + + +def get_collection_coverage( + api: API, request: APIRequest, dataset) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection coverage + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + query_args = {} + format_ = request.format or F_JSON + + # Force response content type and language (en-US only) headers + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'coverage') + + p = load_plugin('provider', collection_def) + except KeyError: + msg = 'collection does not exist' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, format_, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + LOGGER.debug('Processing bbox parameter') + + bbox = request.params.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, format_, + 'InvalidParameterValue', msg) + + query_args['bbox'] = bbox + + LOGGER.debug('Processing bbox-crs parameter') + + bbox_crs = request.params.get('bbox-crs') + if bbox_crs is not None: + query_args['bbox_crs'] = bbox_crs + + LOGGER.debug('Processing datetime parameter') + + datetime_ = request.params.get('datetime') + + try: + datetime_ = validate_datetime( + api.config['resources'][dataset]['extents'], datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + query_args['datetime_'] = datetime_ + query_args['format_'] = format_ + + properties = request.params.get('properties') + if properties: + LOGGER.debug('Processing properties parameter') + query_args['properties'] = [rs for + rs in properties.split(',') if rs] + LOGGER.debug(f"Fields: {query_args['properties']}") + + for a in query_args['properties']: + if a not in p.fields: + msg = 'Invalid field specified' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + if 'subset' in request.params: + LOGGER.debug('Processing subset parameter') + try: + subsets = validate_subset(request.params['subset'] or '') + except (AttributeError, ValueError) as err: + msg = f'Invalid subset: {err}' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + if not set(subsets.keys()).issubset(p.axes): + msg = 'Invalid axis name' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + + query_args['subsets'] = subsets + LOGGER.debug(f"Subsets: {query_args['subsets']}") + + LOGGER.debug('Querying coverage') + try: + data = p.query(**query_args) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + if format_ == mt: # native format + if p.filename is not None: + cd = f'attachment; filename="{p.filename}"' + headers['Content-Disposition'] = cd + + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + elif format_ == F_JSON: + headers['Content-Type'] = 'application/prs.coverage+json' + return headers, HTTPStatus.OK, to_json(data, api.pretty_print) + else: + return api.get_format_exception(request) + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + try: + load_plugin('provider', get_provider_by_type( + collections[k]['providers'], 'coverage')) + except ProviderTypeError: + LOGGER.debug('collection is not coverage based') + continue + + coverage_path = f'/collections/{k}/coverage' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + paths[coverage_path] = { + 'get': { + 'summary': f'Get {title} coverage', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Coverage', + 'parameters': [ + {'$ref': '#/components/parameters/lang'}, + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/bbox'}, + {'$ref': '#/components/parameters/bbox-crs'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + + return [{'name': 'coverages'}], {'paths': paths} diff --git a/pygeoapi/api/environmental_data_retrieval.py b/pygeoapi/api/environmental_data_retrieval.py new file mode 100644 index 0000000..f659cc8 --- /dev/null +++ b/pygeoapi/api/environmental_data_retrieval.py @@ -0,0 +1,332 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from http import HTTPStatus +import logging +from typing import Tuple + +from shapely.errors import WKTReadingError +from shapely.wkt import loads as shapely_loads + +from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.provider.base import ProviderGenericError +from pygeoapi.util import ( + filter_providers_by_type, get_provider_by_type, render_j2_template, + to_json, filter_dict_by_key_value +) + +from . import APIRequest, API, F_HTML, validate_datetime, validate_bbox + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-edr-1/1.0/conf/core' +] + + +def get_collection_edr_query(api: API, request: APIRequest, + dataset, instance, query_type, + location_id=None) -> Tuple[dict, int, str]: + """ + Queries collection EDR + + :param request: APIRequest instance with query params + :param dataset: dataset name + :param instance: instance name + :param query_type: EDR query type + :param location_id: location id of a /location/ query + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + headers = request.get_response_headers(api.default_locale, + **api.api_headers) + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing parameter_names parameter') + parameternames = request.params.get('parameter_names') or [] + if isinstance(parameternames, str): + parameternames = parameternames.split(',') + + bbox = None + if query_type in ['cube', 'locations']: + LOGGER.debug('Processing cube bbox') + try: + bbox = validate_bbox(request.params.get('bbox')) + if not bbox and query_type == 'cube': + raise ValueError('bbox parameter required by cube queries') + except ValueError as err: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + LOGGER.debug('Processing coords parameter') + wkt = request.params.get('coords') + + if wkt: + try: + wkt = shapely_loads(wkt) + except WKTReadingError: + msg = 'invalid coords parameter' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + elif query_type not in ['cube', 'locations']: + msg = 'missing coords parameter' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + within = within_units = None + if query_type == 'radius': + LOGGER.debug('Processing within / within-units parameters') + within = request.params.get('within') + within_units = request.params.get('within-units') + + LOGGER.debug('Processing z parameter') + z = request.params.get('z') + + LOGGER.debug('Loading provider') + try: + p = load_plugin('provider', get_provider_by_type( + collections[dataset]['providers'], 'edr')) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if instance is not None and not p.get_instance(instance): + msg = 'Invalid instance identifier' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, + request.format, 'InvalidParameterValue', msg) + + if query_type not in p.get_query_types(): + msg = 'Unsupported query type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if parameternames and not any((fld in parameternames) + for fld in p.get_fields().keys()): + msg = 'Invalid parameter_names' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + query_args = dict( + query_type=query_type, + instance=instance, + format_=request.format, + datetime_=datetime_, + select_properties=parameternames, + wkt=wkt, + z=z, + bbox=bbox, + within=within, + within_units=within_units, + limit=int(api.config['server']['limit']), + location_id=location_id, + ) + + try: + data = p.query(**query_args) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, + 'collections/edr/query.html', data, + api.default_locale) + else: + content = to_json(data, api.pretty_print) + + return headers, HTTPStatus.OK, content + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + LOGGER.debug('setting up edr endpoints') + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + edr_extension = filter_providers_by_type( + collections[k]['providers'], 'edr') + + if edr_extension: + collection_name_path = f'/collections/{k}' + + ep = load_plugin('provider', edr_extension) + + edr_query_endpoints = [] + + for qt in [qt for qt in ep.get_query_types() if qt != 'locations']: + edr_query_endpoints.append({ + 'path': f'{collection_name_path}/{qt}', + 'qt': qt, + 'op_id': f'query{qt.capitalize()}{k.capitalize()}' + }) + if ep.instances: + edr_query_endpoints.append({ + 'path': f'{collection_name_path}/instances/{{instanceId}}/{qt}', # noqa + 'qt': qt, + 'op_id': f'query{qt.capitalize()}Instance{k.capitalize()}' # noqa + }) + + for eqe in edr_query_endpoints: + if eqe['qt'] == 'cube': + spatial_parameter = 'bbox' + else: + spatial_parameter = f"{eqe['qt']}Coords" + paths[eqe['path']] = { + 'get': { + 'summary': f"query {v['description']} by {eqe['qt']}", + 'description': v['description'], + 'tags': [k], + 'operationId': eqe['op_id'], + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/prs.coverage+json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa + } + } + } + } + } + } + } + if 'locations' in ep.get_query_types(): + paths[f'{collection_name_path}/locations'] = { + 'get': { + 'summary': f"Get pre-defined locations of {v['description']}", # noqa + 'description': v['description'], + 'tags': [k], + 'operationId': f'queryLOCATIONS{k.capitalize()}', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/bbox.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + paths[f'{collection_name_path}/locations/{{locId}}'] = { + 'get': { + 'summary': f"query {v['description']} by location", + 'description': v['description'], + 'tags': [k], + 'operationId': f'queryLOCATIONSBYID{k.capitalize()}', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/locationId.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa + {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, # noqa + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/prs.coverage+json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa + } + } + } + } + } + } + } + + return [{'name': 'edr'}], {'paths': paths} diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py new file mode 100644 index 0000000..4a74606 --- /dev/null +++ b/pygeoapi/api/itemtypes.py @@ -0,0 +1,1610 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +from datetime import datetime +from http import HTTPStatus +import logging +from typing import Any, Tuple, Union, Optional +import urllib.parse + +from pygeofilter.parsers.ecql import parse as parse_ecql_text +from pygeofilter.parsers.cql_json import parse as parse_cql_json +from pyproj.exceptions import CRSError + +from pygeoapi import l10n +from pygeoapi.formatter.base import FormatterSerializationError +from pygeoapi.linked_data import geojson2jsonld +from pygeoapi.plugin import load_plugin, PLUGINS +from pygeoapi.provider.base import ( + ProviderGenericError, ProviderTypeError, SchemaType) + +from pygeoapi.models.cql import CQLModel +from pygeoapi.util import (CrsTransformSpec, filter_providers_by_type, + filter_dict_by_key_value, get_crs_from_uri, + get_provider_by_type, get_supported_crs_list, + modify_pygeofilter, render_j2_template, str2bool, + to_json, transform_bbox) + +from . import ( + APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, + validate_bbox, validate_datetime +) + +LOGGER = logging.getLogger(__name__) + +OGC_RELTYPES_BASE = 'http://www.opengis.net/def/rel/ogc/1.0' + +DEFAULT_CRS_LIST = [ + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', +] + +DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' +DEFAULT_STORAGE_CRS = DEFAULT_CRS + +CONFORMANCE_CLASSES_FEATURES = [ + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/req/oas30', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html', + 'http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson', + 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs', + 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables', + 'http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/queryables-query-parameters', # noqa + 'http://www.opengis.net/spec/ogcapi-features-4/1.0/conf/create-replace-delete', # noqa + 'http://www.opengis.net/spec/ogcapi-features-5/1.0/conf/schemas', + 'http://www.opengis.net/spec/ogcapi-features-5/1.0/req/core-roles-features' +] + +CONFORMANCE_CLASSES_RECORDS = [ + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/sorting', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/opensearch', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-records-1/1.0/conf/html' +] + + +def get_collection_queryables(api: API, request: Union[APIRequest, Any], + dataset=None) -> Tuple[dict, int, str]: + """ + Provide collection queryables + + :param request: A request object + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**api.api_headers) + + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection queryables') + try: + LOGGER.debug('Loading feature provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'feature')) + except ProviderTypeError: + try: + LOGGER.debug('Loading coverage provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'coverage')) # noqa + except ProviderTypeError: + LOGGER.debug('Loading record provider') + p = load_plugin('provider', get_provider_by_type( + api.config['resources'][dataset]['providers'], 'record')) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + queryables = { + 'type': 'object', + 'title': l10n.translate( + api.config['resources'][dataset]['title'], request.locale), + 'properties': {}, + '$schema': 'http://json-schema.org/draft/2019-09/schema', + '$id': f'{api.get_collections_url()}/{dataset}/queryables' + } + + if p.fields: + queryables['properties']['geometry'] = { + '$ref': 'https://geojson.org/schema/Geometry.json', + 'x-ogc-role': 'primary-geometry' + } + + for k, v in p.fields.items(): + show_field = False + if p.properties: + if k in p.properties: + show_field = True + else: + show_field = True + + if show_field: + queryables['properties'][k] = { + 'title': k, + 'type': v['type'] + } + if 'values' in v: + queryables['properties'][k]['enum'] = v['values'] + + if k == p.id_field: + queryables['properties'][k]['x-ogc-role'] = 'id' + if k == p.time_field: + queryables['properties'][k]['x-ogc-role'] = 'primary-instant' # noqa + + if request.format == F_HTML: # render + queryables['title'] = l10n.translate( + api.config['resources'][dataset]['title'], request.locale) + + queryables['collections_path'] = api.get_collections_url() + + content = render_j2_template(api.tpl_config, + 'collections/queryables.html', + queryables, request.locale) + + return headers, HTTPStatus.OK, content + + headers['Content-Type'] = 'application/schema+json' + + return headers, HTTPStatus.OK, to_json(queryables, api.pretty_print) + + +def get_collection_items( + api: API, request: Union[APIRequest, Any], + dataset) -> Tuple[dict, int, str]: + """ + Queries collection + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + + properties = [] + reserved_fieldnames = ['bbox', 'bbox-crs', 'crs', 'f', 'lang', 'limit', + 'offset', 'resulttype', 'datetime', 'sortby', + 'properties', 'skipGeometry', 'q', + 'filter', 'filter-lang', 'filter-crs'] + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing offset parameter') + try: + offset = int(request.params.get('offset')) + if offset < 0: + msg = 'offset value should be positive or zero' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + offset = 0 + except ValueError: + msg = 'offset value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing limit parameter') + try: + limit = int(request.params.get('limit')) + # TODO: We should do more validation, against the min and max + # allowed by the server configuration + if limit <= 0: + msg = 'limit value should be strictly positive' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + limit = int(api.config['server']['limit']) + except ValueError: + msg = 'limit value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + resulttype = request.params.get('resulttype') or 'results' + + LOGGER.debug('Processing bbox parameter') + + bbox = request.params.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('processing q parameter') + q = request.params.get('q') or None + + LOGGER.debug('Loading provider') + + provider_def = None + try: + provider_type = 'feature' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_type = 'record' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + crs_transform_spec = None + if provider_type == 'feature': + # crs query parameter is only available for OGC API - Features + # right now, not for OGC API - Records. + LOGGER.debug('Processing crs parameter') + query_crs_uri = request.params.get('crs') + try: + crs_transform_spec = create_crs_transform_spec( + provider_def, query_crs_uri, + ) + except (ValueError, CRSError) as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + set_content_crs_header(headers, provider_def, query_crs_uri) + + LOGGER.debug('Processing bbox-crs parameter') + bbox_crs = request.params.get('bbox-crs') + if bbox_crs is not None: + # Validate bbox-crs parameter + if len(bbox) == 0: + msg = 'bbox-crs specified without bbox parameter' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + + if len(bbox_crs) == 0: + msg = 'bbox-crs specified but is empty' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + + supported_crs_list = get_supported_crs_list(provider_def, DEFAULT_CRS_LIST) # noqa + if bbox_crs not in supported_crs_list: + msg = f'bbox-crs {bbox_crs} not supported for this collection' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + elif len(bbox) > 0: + # bbox but no bbox-crs parm: assume bbox is in default CRS + bbox_crs = DEFAULT_CRS + + # Transform bbox to storageCRS + # when bbox-crs different from storageCRS. + if len(bbox) > 0: + try: + # Get a pyproj CRS instance for the Collection's Storage CRS + storage_crs = provider_def.get('storage_crs', DEFAULT_STORAGE_CRS) # noqa + + # Do the (optional) Transform to the Storage CRS + bbox = transform_bbox(bbox, bbox_crs, storage_crs) + except CRSError as e: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', str(e)) + + LOGGER.debug('processing property parameters') + for k, v in request.params.items(): + if k not in reserved_fieldnames and k in list(p.fields.keys()): + LOGGER.debug(f'Adding property filter {k}={v}') + properties.append((k, v)) + + LOGGER.debug('processing sort parameter') + val = request.params.get('sortby') + + if val is not None: + sortby = [] + sorts = val.split(',') + for s in sorts: + prop = s + order = '+' + if s[0] in ['+', '-']: + order = s[0] + prop = s[1:] + + if prop not in p.fields.keys(): + msg = 'bad sort property' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + sortby.append({'property': prop, 'order': order}) + else: + sortby = [] + + LOGGER.debug('processing properties parameter') + val = request.params.get('properties') + + if val is not None: + select_properties = val.split(',') + properties_to_check = set(p.properties) | set(p.fields.keys()) + + if (len(list(set(select_properties) - + set(properties_to_check))) > 0): + msg = 'unknown properties specified' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + select_properties = [] + + LOGGER.debug('processing skipGeometry parameter') + val = request.params.get('skipGeometry') + if val is not None: + skip_geometry = str2bool(val) + else: + skip_geometry = False + + LOGGER.debug('Processing filter-crs parameter') + filter_crs_uri = request.params.get('filter-crs', DEFAULT_CRS) + LOGGER.debug('processing filter parameter') + cql_text = request.params.get('filter') + if cql_text is not None: + try: + filter_ = parse_ecql_text(cql_text) + filter_ = modify_pygeofilter( + filter_, + filter_crs_uri=filter_crs_uri, + storage_crs_uri=provider_def.get('storage_crs'), + geometry_column_name=provider_def.get('geom_field'), + ) + except Exception as err: + LOGGER.error(err) + msg = f'Bad CQL string : {cql_text}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + filter_ = None + + LOGGER.debug('Processing filter-lang parameter') + filter_lang = request.params.get('filter-lang') + # Currently only cql-text is handled, but it is optional + if filter_lang not in [None, 'cql-text']: + msg = 'Invalid filter language' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + # Get provider locale (if any) + prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) + + LOGGER.debug('Querying provider') + LOGGER.debug(f'offset: {offset}') + LOGGER.debug(f'limit: {limit}') + LOGGER.debug(f'resulttype: {resulttype}') + LOGGER.debug(f'sortby: {sortby}') + LOGGER.debug(f'bbox: {bbox}') + if provider_type == 'feature': + LOGGER.debug(f'crs: {query_crs_uri}') + LOGGER.debug(f'datetime: {datetime_}') + LOGGER.debug(f'properties: {properties}') + LOGGER.debug(f'select properties: {select_properties}') + LOGGER.debug(f'skipGeometry: {skip_geometry}') + LOGGER.debug(f'language: {prv_locale}') + LOGGER.debug(f'q: {q}') + LOGGER.debug(f'cql_text: {cql_text}') + LOGGER.debug(f'filter_: {filter_}') + LOGGER.debug(f'filter-lang: {filter_lang}') + LOGGER.debug(f'filter-crs: {filter_crs_uri}') + + try: + content = p.query(offset=offset, limit=limit, + resulttype=resulttype, bbox=bbox, + datetime_=datetime_, properties=properties, + sortby=sortby, skip_geometry=skip_geometry, + select_properties=select_properties, + crs_transform_spec=crs_transform_spec, + q=q, language=prv_locale, filterq=filter_) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + serialized_query_params = '' + for k, v in request.params.items(): + if k not in ('f', 'offset'): + serialized_query_params += '&' + serialized_query_params += urllib.parse.quote(k, safe='') + serialized_query_params += '=' + serialized_query_params += urllib.parse.quote(str(v), safe=',') + + # TODO: translate titles + uri = f'{api.get_collections_url()}/{dataset}/items' + content['links'] = [{ + 'type': 'application/geo+json', + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as GeoJSON', + 'href': f'{uri}?f={F_JSON}{serialized_query_params}' + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{uri}?f={F_JSONLD}{serialized_query_params}' + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{uri}?f={F_HTML}{serialized_query_params}' + }] + + if offset > 0: + prev = max(0, offset - limit) + content['links'].append( + { + 'type': 'application/geo+json', + 'rel': 'prev', + 'title': 'items (prev)', + 'href': f'{uri}?offset={prev}{serialized_query_params}' + }) + + if 'numberMatched' in content: + if content['numberMatched'] > (limit + offset): + next_ = offset + limit + next_href = f'{uri}?offset={next_}{serialized_query_params}' + content['links'].append( + { + 'type': 'application/geo+json', + 'rel': 'next', + 'title': 'items (next)', + 'href': next_href + }) + + content['links'].append( + { + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate( + collections[dataset]['title'], request.locale), + 'rel': 'collection', + 'href': uri + }) + + content['timeStamp'] = datetime.utcnow().strftime( + '%Y-%m-%dT%H:%M:%S.%fZ') + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + if request.format == F_HTML: # render + # For constructing proper URIs to items + + content['items_path'] = uri + content['dataset_path'] = '/'.join(uri.split('/')[:-1]) + content['collections_path'] = api.get_collections_url() + + content['offset'] = offset + + content['id_field'] = p.id_field + if p.uri_field is not None: + content['uri_field'] = p.uri_field + if p.title_field is not None: + content['title_field'] = l10n.translate(p.title_field, + request.locale) + # If title exists, use it as id in html templates + content['id_field'] = content['title_field'] + content = render_j2_template(api.tpl_config, + 'collections/items/index.html', + content, request.locale) + return headers, HTTPStatus.OK, content + elif request.format == 'csv': # render + formatter = load_plugin('formatter', + {'name': 'CSV', 'geom': True}) + + try: + content = formatter.write( + data=content, + options={ + 'provider_def': get_provider_by_type( + collections[dataset]['providers'], + 'feature') + } + ) + except FormatterSerializationError as err: + LOGGER.error(err) + msg = 'Error serializing output' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format, + 'NoApplicableCode', msg) + + headers['Content-Type'] = formatter.mimetype + + if p.filename is None: + filename = f'{dataset}.csv' + else: + filename = f'{p.filename}' + + cd = f'attachment; filename="{filename}"' + headers['Content-Disposition'] = cd + + return headers, HTTPStatus.OK, content + + elif request.format == F_JSONLD: + content = geojson2jsonld( + api, content, dataset, id_field=(p.uri_field or 'id') + ) + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +def post_collection_items( + api: API, request: APIRequest, dataset) -> Tuple[dict, int, str]: + """ + Queries collection or filter an item + + :param request: A request object + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + request_headers = request.headers + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + properties = [] + reserved_fieldnames = ['bbox', 'f', 'limit', 'offset', + 'resulttype', 'datetime', 'sortby', + 'properties', 'skipGeometry', 'q', + 'filter-lang', 'filter-crs'] + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Invalid collection' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Processing offset parameter') + try: + offset = int(request.params.get('offset')) + if offset < 0: + msg = 'offset value should be positive or zero' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + offset = 0 + except ValueError: + msg = 'offset value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing limit parameter') + try: + limit = int(request.params.get('limit')) + # TODO: We should do more validation, against the min and max + # allowed by the server configuration + if limit <= 0: + msg = 'limit value should be strictly positive' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except TypeError as err: + LOGGER.warning(err) + limit = int(api.config['server']['limit']) + except ValueError: + msg = 'limit value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + resulttype = request.params.get('resulttype') or 'results' + + LOGGER.debug('Processing bbox parameter') + + bbox = request.params.get('bbox') + + if bbox is None: + bbox = [] + else: + try: + bbox = validate_bbox(bbox) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + datetime_ = validate_datetime(collections[dataset]['extents'], + datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('processing q parameter') + val = request.params.get('q') + + q = None + if val is not None: + q = val + + LOGGER.debug('Loading provider') + + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'feature') + except ProviderTypeError: + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'record') + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'NoApplicableCode', msg) + + try: + p = load_plugin('provider', provider_def) + except ProviderGenericError as err: + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + LOGGER.debug('processing property parameters') + for k, v in request.params.items(): + if k not in reserved_fieldnames and k not in p.fields.keys(): + msg = f'unknown query parameter: {k}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + elif k not in reserved_fieldnames and k in p.fields.keys(): + LOGGER.debug(f'Add property filter {k}={v}') + properties.append((k, v)) + + LOGGER.debug('processing sort parameter') + val = request.params.get('sortby') + + if val is not None: + sortby = [] + sorts = val.split(',') + for s in sorts: + prop = s + order = '+' + if s[0] in ['+', '-']: + order = s[0] + prop = s[1:] + + if prop not in p.fields.keys(): + msg = 'bad sort property' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + sortby.append({'property': prop, 'order': order}) + else: + sortby = [] + + LOGGER.debug('processing properties parameter') + val = request.params.get('properties') + + if val is not None: + select_properties = val.split(',') + properties_to_check = set(p.properties) | set(p.fields.keys()) + + if (len(list(set(select_properties) - + set(properties_to_check))) > 0): + msg = 'unknown properties specified' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + select_properties = [] + + LOGGER.debug('processing skipGeometry parameter') + val = request.params.get('skipGeometry') + if val is not None: + skip_geometry = str2bool(val) + else: + skip_geometry = False + + LOGGER.debug('Processing filter-crs parameter') + filter_crs = request.params.get('filter-crs', DEFAULT_CRS) + LOGGER.debug('Processing filter-lang parameter') + filter_lang = request.params.get('filter-lang') + if filter_lang != 'cql-json': # @TODO add check from the configuration + msg = 'Invalid filter language' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Querying provider') + LOGGER.debug(f'offset: {offset}') + LOGGER.debug(f'limit: {limit}') + LOGGER.debug(f'resulttype: {resulttype}') + LOGGER.debug(f'sortby: {sortby}') + LOGGER.debug(f'bbox: {bbox}') + LOGGER.debug(f'datetime: {datetime_}') + LOGGER.debug(f'properties: {select_properties}') + LOGGER.debug(f'skipGeometry: {skip_geometry}') + LOGGER.debug(f'q: {q}') + LOGGER.debug(f'filter-lang: {filter_lang}') + LOGGER.debug(f'filter-crs: {filter_crs}') + + LOGGER.debug('Processing headers') + + LOGGER.debug('Processing request content-type header') + if (request_headers.get( + 'Content-Type') or request_headers.get( + 'content-type')) != 'application/query-cql-json': + msg = ('Invalid body content-type') + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidHeaderValue', msg) + + LOGGER.debug('Processing body') + + if not request.data: + msg = 'missing request data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'MissingParameterValue', msg) + + filter_ = None + try: + # Parse bytes data, if applicable + data = request.data.decode() + LOGGER.debug(data) + except UnicodeDecodeError as err: + LOGGER.error(err) + msg = 'Unicode error in data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + # FIXME: remove testing backend in use once CQL support is normalized + if p.name == 'PostgreSQL': + LOGGER.debug('processing PostgreSQL CQL_JSON data') + try: + filter_ = parse_cql_json(data) + filter_ = modify_pygeofilter( + filter_, + filter_crs_uri=filter_crs, + storage_crs_uri=provider_def.get('storage_crs'), + geometry_column_name=provider_def.get('geom_field') + ) + except Exception as err: + LOGGER.error(err) + msg = f'Bad CQL string : {data}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + else: + LOGGER.debug('processing Elasticsearch CQL_JSON data') + try: + filter_ = CQLModel.parse_raw(data) + except Exception as err: + LOGGER.error(err) + msg = f'Bad CQL string : {data}' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + try: + content = p.query(offset=offset, limit=limit, + resulttype=resulttype, bbox=bbox, + datetime_=datetime_, properties=properties, + sortby=sortby, + select_properties=select_properties, + skip_geometry=skip_geometry, + q=q, + filterq=filter_) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +def manage_collection_item( + api: API, request: APIRequest, + action, dataset, identifier=None) -> Tuple[dict, int, str]: + """ + Adds an item to a collection + + :param request: A request object + :param action: an action among 'create', 'update', 'delete', 'options' + :param dataset: dataset name + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(PLUGINS['formatter'].keys()): + return api.get_format_exception(request) + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'feature') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_def = get_provider_by_type( + collections[dataset]['providers'], 'record') + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action == 'options': + headers['Allow'] = 'HEAD, GET' + if p.editable: + if identifier is None: + headers['Allow'] += ', POST' + else: + headers['Allow'] += ', PUT, DELETE' + return headers, HTTPStatus.OK, '' + + if not p.editable: + msg = 'Collection is not editable' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action in ['create', 'update'] and not request.data: + msg = 'No data found' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + if action == 'create': + LOGGER.debug('Creating item') + try: + identifier = p.create(request.data) + except TypeError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + headers['Location'] = f'{api.get_collections_url()}/{dataset}/items/{identifier}' # noqa + + return headers, HTTPStatus.CREATED, '' + + if action == 'update': + LOGGER.debug('Updating item') + try: + _ = p.update(identifier, request.data) + except TypeError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + return headers, HTTPStatus.NO_CONTENT, '' + + if action == 'delete': + LOGGER.debug('Deleting item') + try: + _ = p.delete(identifier) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + return headers, HTTPStatus.OK, '' + + +def get_collection_item(api: API, request: APIRequest, + dataset, identifier) -> Tuple[dict, int, str]: + """ + Get a single collection item + + :param request: A request object + :param dataset: dataset name + :param identifier: item identifier + + :returns: tuple of headers, status code, content + """ + + # Set Content-Language to system locale until provider locale + # has been determined + headers = request.get_response_headers(SYSTEM_LOCALE, **api.api_headers) + + LOGGER.debug('Processing query parameters') + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + + try: + provider_type = 'feature' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + try: + provider_type = 'record' + provider_def = get_provider_by_type( + collections[dataset]['providers'], provider_type) + p = load_plugin('provider', provider_def) + except ProviderTypeError: + msg = 'Invalid provider type' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + crs_transform_spec = None + if provider_type == 'feature': + # crs query parameter is only available for OGC API - Features + # right now, not for OGC API - Records. + LOGGER.debug('Processing crs parameter') + query_crs_uri = request.params.get('crs') + try: + crs_transform_spec = create_crs_transform_spec( + provider_def, query_crs_uri, + ) + except (ValueError, CRSError) as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + set_content_crs_header(headers, provider_def, query_crs_uri) + + # Get provider language (if any) + prv_locale = l10n.get_plugin_locale(provider_def, request.raw_locale) + + try: + LOGGER.debug(f'Fetching id {identifier}') + content = p.get( + identifier, + language=prv_locale, + crs_transform_spec=crs_transform_spec, + ) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + if content is None: + msg = 'identifier not found' + return api.get_exception(HTTPStatus.BAD_REQUEST, headers, + request.format, 'NotFound', msg) + + uri = content['properties'].get(p.uri_field) if p.uri_field else \ + f'{api.get_collections_url()}/{dataset}/items/{identifier}' + + if 'links' not in content: + content['links'] = [] + + content['links'].extend([{ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'root', + 'title': 'The landing page of this server as JSON', + 'href': f"{api.base_url}?f={F_JSON}" + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'root', + 'title': 'The landing page of this server as HTML', + 'href': f"{api.base_url}?f={F_HTML}" + }, { + 'rel': request.get_linkrel(F_JSON), + 'type': 'application/geo+json', + 'title': 'This document as GeoJSON', + 'href': f'{uri}?f={F_JSON}' + }, { + 'rel': request.get_linkrel(F_JSONLD), + 'type': FORMAT_TYPES[F_JSONLD], + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{uri}?f={F_JSONLD}' + }, { + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': 'This document as HTML', + 'href': f'{uri}?f={F_HTML}' + }, { + 'rel': 'collection', + 'type': FORMAT_TYPES[F_JSON], + 'title': l10n.translate(collections[dataset]['title'], + request.locale), + 'href': f'{api.get_collections_url()}/{dataset}' + }]) + + link_request_format = ( + request.format if request.format is not None else F_JSON + ) + if 'prev' in content: + content['links'].append({ + 'rel': 'prev', + 'type': FORMAT_TYPES[link_request_format], + 'href': f"{api.get_collections_url()}/{dataset}/items/{content['prev']}?f={link_request_format}" # noqa + }) + if 'next' in content: + content['links'].append({ + 'rel': 'next', + 'type': FORMAT_TYPES[link_request_format], + 'href': f"{api.get_collections_url()}/{dataset}/items/{content['next']}?f={link_request_format}" # noqa + }) + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + if request.format == F_HTML: # render + content['title'] = l10n.translate(collections[dataset]['title'], + request.locale) + content['id_field'] = p.id_field + if p.uri_field is not None: + content['uri_field'] = p.uri_field + if p.title_field is not None: + content['title_field'] = l10n.translate(p.title_field, + request.locale) + content['collections_path'] = api.get_collections_url() + + content = render_j2_template(api.tpl_config, + 'collections/items/item.html', + content, request.locale) + return headers, HTTPStatus.OK, content + + elif request.format == F_JSONLD: + content = geojson2jsonld( + api, content, dataset, uri, (p.uri_field or 'id') + ) + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +@staticmethod +def create_crs_transform_spec( + config: dict, query_crs_uri: Optional[str] = None) -> Union[None, CrsTransformSpec]: # noqa + """ + Create a `CrsTransformSpec` instance based on provider config and + *crs* query parameter. + + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system (CRS) specified in query parameter (if specified). + :type query_crs_uri: str, optional + + :raises ValueError: Error raised if the CRS specified in the query + parameter is not in the list of supported CRSs of the provider. + :raises `CRSError`: Error raised if no CRS could be identified from the + query *crs* parameter (URI). + + :returns: `CrsTransformSpec` instance if the CRS specified in query + parameter differs from the storage CRS, else `None`. + :rtype: Union[None, CrsTransformSpec] + """ + + # Get storage/default CRS for Collection. + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + + if not query_crs_uri: + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + query_crs_uri = storage_crs_uri + else: + query_crs_uri = DEFAULT_CRS + LOGGER.debug(f'no crs parameter, using default: {query_crs_uri}') + + supported_crs_list = get_supported_crs_list(config, DEFAULT_CRS_LIST) + # Check that the crs specified by the query parameter is supported. + if query_crs_uri not in supported_crs_list: + raise ValueError( + f'CRS {query_crs_uri!r} not supported for this ' + 'collection. List of supported CRSs: ' + f'{", ".join(supported_crs_list)}.' + ) + crs_out = get_crs_from_uri(query_crs_uri) + + storage_crs = get_crs_from_uri(storage_crs_uri) + # Check if the crs specified in query parameter differs from the + # storage crs. + if str(storage_crs) != str(crs_out): + LOGGER.debug( + f'CRS transformation: {storage_crs} -> {crs_out}' + ) + return CrsTransformSpec( + source_crs_uri=storage_crs_uri, + source_crs_wkt=storage_crs.to_wkt(), + target_crs_uri=query_crs_uri, + target_crs_wkt=crs_out.to_wkt(), + ) + else: + LOGGER.debug('No CRS transformation') + return None + + +@staticmethod +def set_content_crs_header( + headers: dict, config: dict, query_crs_uri: Optional[str] = None): + """ + Set the *Content-Crs* header in responses from providers of Feature type. + + :param headers: Response headers dictionary. + :type headers: dict + :param config: Provider config dictionary. + :type config: dict + :param query_crs_uri: Uniform resource identifier of the coordinate + reference system specified in query parameter (if specified). + :type query_crs_uri: str, optional + + :returns: None + """ + + if query_crs_uri: + content_crs_uri = query_crs_uri + else: + # If empty use default CRS + storage_crs_uri = config.get('storage_crs', DEFAULT_STORAGE_CRS) + if storage_crs_uri in DEFAULT_CRS_LIST: + # Could be that storageCRS is one of the defaults like + # http://www.opengis.net/def/crs/OGC/1.3/CRS84h + content_crs_uri = storage_crs_uri + else: + content_crs_uri = DEFAULT_CRS + + headers['Content-Crs'] = f'<{content_crs_uri}>' + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + properties = { + 'name': 'properties', + 'in': 'query', + 'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.', # noqa + 'required': False, + 'style': 'form', + 'explode': False, + 'schema': { + 'type': 'array', + 'items': { + 'type': 'string' + } + } + } + + LOGGER.debug('setting up collection endpoints') + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + try: + ptype = None + + if filter_providers_by_type( + collections[k]['providers'], 'feature'): + ptype = 'feature' + + if filter_providers_by_type( + collections[k]['providers'], 'record'): + ptype = 'record' + + p = load_plugin('provider', get_provider_by_type( + collections[k]['providers'], ptype)) + + collection_name_path = f'/collections/{k}' + items_path = f'/collections/{k}/items' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + coll_properties = deepcopy(properties) + + coll_properties['schema']['items']['enum'] = list(p.fields.keys()) + + paths[items_path] = { + 'get': { + 'summary': f'Get {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Features', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'}, + {'$ref': '#/components/parameters/bbox'}, + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/limit"}, # noqa + {'$ref': '#/components/parameters/crs'}, # noqa + {'$ref': '#/components/parameters/bbox-crs'}, + coll_properties, + {'$ref': '#/components/parameters/vendorSpecificParameters'}, # noqa + {'$ref': '#/components/parameters/skipGeometry'}, + {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/sortby.yaml"}, # noqa + {'$ref': '#/components/parameters/offset'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + 'options': { + 'summary': f'Options for {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'options{k.capitalize()}Features', + 'responses': { + '200': {'description': 'options response'} + } + } + } + + if p.editable: + LOGGER.debug('Provider is editable; adding post') + + paths[items_path]['post'] = { + 'summary': f'Add {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'add{k.capitalize()}Features', + 'requestBody': { + 'description': 'Adds item to collection', + 'content': { + 'application/geo+json': { + 'schema': {} + } + }, + 'required': True + }, + 'responses': { + '201': {'description': 'Successful creation'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + try: + schema_ref = p.get_schema(SchemaType.create) + paths[items_path]['post']['requestBody']['content'][schema_ref[0]] = { # noqa + 'schema': schema_ref[1] + } + except Exception as err: + LOGGER.debug(err) + + if ptype == 'record': + paths[items_path]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/q.yaml"}) + if p.fields: + schema_path = f'{collection_name_path}/schema' + + paths[schema_path] = { + 'get': { + 'summary': f'Get {title} schema', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Schema', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Queryables'}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + + queryables_path = f'{collection_name_path}/queryables' + + paths[queryables_path] = { + 'get': { + 'summary': f'Get {title} queryables', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Queryables', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Queryables'}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + + if p.time_field is not None: + paths[items_path]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa + + for field, type_ in p.fields.items(): + + if p.properties and field not in p.properties: + LOGGER.debug('Provider specified not to advertise property') # noqa + continue + + if field == 'q' and ptype == 'record': + LOGGER.debug('q parameter already declared, skipping') + continue + + if type_ == 'date': + schema = { + 'type': 'string', + 'format': 'date' + } + elif type_ == 'float': + schema = { + 'type': 'number', + 'format': 'float' + } + elif type_ == 'long': + schema = { + 'type': 'integer', + 'format': 'int64' + } + else: + schema = type_ + + path_ = f'{collection_name_path}/items' + paths[path_]['get']['parameters'].append({ + 'name': field, + 'in': 'query', + 'required': False, + 'schema': schema, + 'style': 'form', + 'explode': False + }) + + paths[f'{collection_name_path}/items/{{featureId}}'] = { + 'get': { + 'summary': f'Get {title} item by id', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Feature', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa + {'$ref': '#/components/parameters/crs'}, # noqa + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Feature"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + 'options': { + 'summary': f'Options for {title} item by id', + 'description': description, + 'tags': [k], + 'operationId': f'options{k.capitalize()}Feature', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa + ], + 'responses': { + '200': {'description': 'options response'} + } + } + } + + try: + schema_ref = p.get_schema() + paths[f'{collection_name_path}/items/{{featureId}}']['get']['responses']['200'] = { # noqa + 'content': { + schema_ref[0]: { + 'schema': schema_ref[1] + } + } + } + except Exception as err: + LOGGER.debug(err) + + if p.editable: + LOGGER.debug('Provider is editable; adding put/delete') + put_path = f'{collection_name_path}/items/{{featureId}}' # noqa + paths[put_path]['put'] = { # noqa + 'summary': f'Update {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'update{k.capitalize()}Features', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa + ], + 'requestBody': { + 'description': 'Updates item in collection', + 'content': { + 'application/geo+json': { + 'schema': {} + } + }, + 'required': True + }, + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + try: + schema_ref = p.get_schema(SchemaType.replace) + paths[put_path]['put']['requestBody']['content'][schema_ref[0]] = { # noqa + 'schema': schema_ref[1] + } + except Exception as err: + LOGGER.debug(err) + + paths[f'{collection_name_path}/items/{{featureId}}']['delete'] = { # noqa + 'summary': f'Delete {title} items', + 'description': description, + 'tags': [k], + 'operationId': f'delete{k.capitalize()}Features', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa + ], + 'responses': { + '200': {'description': 'Successful delete'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + + except ProviderTypeError: + LOGGER.debug('collection is not feature/item based') + + return [{'name': 'records'}, {'name': 'features'}], {'paths': paths} diff --git a/pygeoapi/api/maps.py b/pygeoapi/api/maps.py new file mode 100644 index 0000000..03866fb --- /dev/null +++ b/pygeoapi/api/maps.py @@ -0,0 +1,340 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +from http import HTTPStatus +import logging +from typing import Tuple + +from pygeoapi.openapi import get_oas_30_parameters +from pygeoapi.plugin import load_plugin +from pygeoapi.provider.base import ProviderGenericError +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_providers_by_type, + filter_dict_by_key_value +) + +from . import APIRequest, API, validate_datetime + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-maps-1/1.0/conf/core' +] + + +def get_collection_map(api: API, request: APIRequest, + dataset, style=None) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection map + + :param request: A request object + :param dataset: dataset name + :param style: style name + + :returns: tuple of headers, status code, content + """ + + query_args = { + 'crs': 'CRS84' + } + + format_ = request.format or 'png' + headers = request.get_response_headers(**api.api_headers) + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'map') + + p = load_plugin('provider', collection_def) + except KeyError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'collection does not exist' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.NOT_FOUND, to_json( + exception, api.pretty_print) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + query_args['format_'] = request.params.get('f', 'png') + query_args['style'] = style + query_args['crs'] = request.params.get('bbox-crs', 4326) + query_args['transparent'] = request.params.get('transparent', True) + + try: + query_args['width'] = int(request.params.get('width', 500)) + query_args['height'] = int(request.params.get('height', 300)) + except ValueError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid width/height' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, api.pretty_print) + + LOGGER.debug('Processing bbox parameter') + try: + bbox = request.params.get('bbox').split(',') + if len(bbox) != 4: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'bbox values should be minx,miny,maxx,maxy' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, api.pretty_print) + except AttributeError: + bbox = api.config['resources'][dataset]['extents']['spatial']['bbox'] # noqa + try: + query_args['bbox'] = [float(c) for c in bbox] + except ValueError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'bbox values must be numbers' + } + headers['Content-type'] = 'application/json' + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + exception, api.pretty_print) + + LOGGER.debug('Processing datetime parameter') + datetime_ = request.params.get('datetime') + try: + query_args['datetime_'] = validate_datetime( + api.config['resources'][dataset]['extents'], datetime_) + except ValueError as err: + msg = str(err) + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + LOGGER.debug('Generating map') + try: + data = p.query(**query_args) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + + if format_ == mt: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + elif format_ in [None, 'html']: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + else: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid format parameter' + } + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + data, api.pretty_print) + + +def get_collection_map_legend(api: API, request: APIRequest, + dataset, style=None) -> Tuple[dict, int, str]: + """ + Returns a subset of a collection map legend + + :param request: A request object + :param dataset: dataset name + :param style: style name + + :returns: tuple of headers, status code, content + """ + + format_ = 'png' + headers = request.get_response_headers(**api.api_headers) + LOGGER.debug('Processing query parameters') + + LOGGER.debug('Loading provider') + try: + collection_def = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'map') + + p = load_plugin('provider', collection_def) + except KeyError: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'collection does not exist' + } + LOGGER.error(exception) + return headers, HTTPStatus.NOT_FOUND, to_json( + exception, api.pretty_print) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + LOGGER.debug('Generating legend') + try: + data = p.get_legend(style, request.params.get('f', 'png')) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + mt = collection_def['format']['name'] + + if format_ == mt: + headers['Content-Type'] = collection_def['format']['mimetype'] + return headers, HTTPStatus.OK, data + else: + exception = { + 'code': 'InvalidParameterValue', + 'description': 'invalid format parameter' + } + LOGGER.error(exception) + return headers, HTTPStatus.BAD_REQUEST, to_json( + data, api.pretty_print) + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + LOGGER.debug('setting up maps endpoints') + + paths = {} + + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + parameters = get_oas_30_parameters(cfg, locale) + for k, v in get_visible_collections(cfg).items(): + map_extension = filter_providers_by_type( + collections[k]['providers'], 'map') + + if map_extension: + mp = load_plugin('provider', map_extension) + + map_f = deepcopy(parameters['f']) + map_f['schema']['enum'] = [map_extension['format']['name']] + map_f['schema']['default'] = map_extension['format']['name'] + + pth = f'/collections/{k}/map' + paths[pth] = { + 'get': { + 'summary': 'Get map', + 'description': f"{v['description']} map", + 'tags': [k], + 'operationId': 'getMap', + 'parameters': [ + {'$ref': '#/components/parameters/bbox'}, + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa + { + 'name': 'width', + 'in': 'query', + 'description': 'Response image width', + 'required': False, + 'schema': { + 'type': 'integer', + }, + 'style': 'form', + 'explode': False + }, + { + 'name': 'height', + 'in': 'query', + 'description': 'Response image height', + 'required': False, + 'schema': { + 'type': 'integer', + }, + 'style': 'form', + 'explode': False + }, + { + 'name': 'transparent', + 'in': 'query', + 'description': 'Background transparency of map (default=true).', # noqa + 'required': False, + 'schema': { + 'type': 'boolean', + 'default': True, + }, + 'style': 'form', + 'explode': False + }, + {'$ref': '#/components/parameters/bbox-crs-epsg'}, + map_f + ], + 'responses': { + '200': { + 'description': 'Response', + 'content': { + 'application/json': {} + } + }, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa + } + } + } + if mp.time_field is not None: + paths[pth]['get']['parameters'].append( + {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa + + return [{'name': 'maps'}], {'paths': paths} diff --git a/pygeoapi/api/processes.py b/pygeoapi/api/processes.py new file mode 100644 index 0000000..6cc7a71 --- /dev/null +++ b/pygeoapi/api/processes.py @@ -0,0 +1,740 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from copy import deepcopy +from datetime import datetime, timezone +from http import HTTPStatus +import json +import logging +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.util import ( + json_serial, render_j2_template, JobStatus, RequestedProcessExecutionMode, + to_json, DATETIME_FORMAT) +from pygeoapi.process.base import ( + JobNotFoundError, JobResultNotFoundError, ProcessorExecuteError +) +from pygeoapi.process.manager.base import get_manager, Subscriber + +from . import ( + APIRequest, API, SYSTEM_LOCALE, F_JSON, FORMAT_TYPES, F_HTML, F_JSONLD, +) + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/ogc-process-description', # noqa + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/json', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-processes-1/1.0/conf/callback' +] + + +def describe_processes(api: API, request: APIRequest, + process=None) -> Tuple[dict, int, str]: + """ + Provide processes metadata + + :param request: A request object + :param process: process identifier, defaults to None to obtain + information about all processes + + :returns: tuple of headers, status code, content + """ + + processes = [] + + headers = request.get_response_headers(**api.api_headers) + + if process is not None: + if process not in api.manager.processes.keys(): + msg = 'Identifier not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchProcess', msg) + + if len(api.manager.processes) > 0: + if process is not None: + relevant_processes = [process] + else: + LOGGER.debug('Processing limit parameter') + try: + limit = int(request.params.get('limit')) + + if limit <= 0: + msg = 'limit value should be strictly positive' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + relevant_processes = list(api.manager.processes)[:limit] + except TypeError: + LOGGER.debug('returning all processes') + relevant_processes = api.manager.processes.keys() + except ValueError: + msg = 'limit value should be an integer' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + for key in relevant_processes: + p = api.manager.get_processor(key) + p2 = l10n.translate_struct(deepcopy(p.metadata), + request.locale) + p2['id'] = key + + if process is None: + p2.pop('inputs') + p2.pop('outputs') + p2.pop('example', None) + + p2['jobControlOptions'] = ['sync-execute'] + if api.manager.is_async: + p2['jobControlOptions'].append('async-execute') + + p2['outputTransmission'] = ['value'] + p2['links'] = p2.get('links', []) + + jobs_url = f"{api.base_url}/jobs" + process_url = f"{api.base_url}/processes/{key}" + + # TODO translation support + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'href': f'{process_url}?f={F_JSON}', + 'title': 'Process description as JSON', + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'href': f'{process_url}?f={F_HTML}', + 'title': 'Process description as HTML', + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_HTML], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', + 'href': f'{jobs_url}?f={F_HTML}', + 'title': 'jobs for this process as HTML', + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/job-list', + 'href': f'{jobs_url}?f={F_JSON}', + 'title': 'jobs for this process as JSON', + 'hreflang': api.default_locale + } + p2['links'].append(link) + + link = { + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/execute', + 'href': f'{process_url}/execution?f={F_JSON}', + 'title': 'Execution for this process as JSON', + 'hreflang': api.default_locale + } + p2['links'].append(link) + + processes.append(p2) + + if process is not None: + response = processes[0] + else: + process_url = f"{api.base_url}/processes" + response = { + 'processes': processes, + 'links': [{ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{process_url}?f={F_JSON}' + }, { + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{process_url}?f={F_JSONLD}' + }, { + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{process_url}?f={F_HTML}' + }] + } + + if request.format == F_HTML: # render + if process is not None: + response = render_j2_template(api.tpl_config, + 'processes/process.html', + response, request.locale) + else: + response = render_j2_template(api.tpl_config, + 'processes/index.html', response, + request.locale) + + return headers, HTTPStatus.OK, response + + return headers, HTTPStatus.OK, to_json(response, api.pretty_print) + + +# TODO: get_jobs doesn't have tests +def get_jobs(api: API, request: APIRequest, + job_id=None) -> Tuple[dict, int, str]: + """ + Get process jobs + + :param request: A request object + :param job_id: id of job + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + if job_id is None: + jobs = sorted(api.manager.get_jobs(), + key=lambda k: k['job_start_datetime'], + reverse=True) + else: + try: + jobs = [api.manager.get_job(job_id)] + except JobNotFoundError: + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, + 'InvalidParameterValue', job_id) + + serialized_jobs = { + 'jobs': [], + 'links': [{ + 'href': f"{api.base_url}/jobs?f={F_HTML}", + 'rel': request.get_linkrel(F_HTML), + 'type': FORMAT_TYPES[F_HTML], + 'title': 'Jobs list as HTML' + }, { + 'href': f"{api.base_url}/jobs?f={F_JSON}", + 'rel': request.get_linkrel(F_JSON), + 'type': FORMAT_TYPES[F_JSON], + 'title': 'Jobs list as JSON' + }] + } + for job_ in jobs: + job2 = { + 'type': 'process', + 'processID': job_['process_id'], + 'jobID': job_['identifier'], + 'status': job_['status'], + 'message': job_['message'], + 'progress': job_['progress'], + 'parameters': job_.get('parameters'), + 'job_start_datetime': job_['job_start_datetime'], + 'job_end_datetime': job_['job_end_datetime'] + } + + # TODO: translate + if JobStatus[job_['status']] in ( + JobStatus.successful, JobStatus.running, JobStatus.accepted): + + job_result_url = f"{api.base_url}/jobs/{job_['identifier']}/results" # noqa + + job2['links'] = [{ + 'href': f'{job_result_url}?f={F_HTML}', + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', + 'type': FORMAT_TYPES[F_HTML], + 'title': f'results of job {job_id} as HTML' + }, { + 'href': f'{job_result_url}?f={F_JSON}', + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', + 'type': FORMAT_TYPES[F_JSON], + 'title': f'results of job {job_id} as JSON' + }] + + if job_['mimetype'] not in (FORMAT_TYPES[F_JSON], + FORMAT_TYPES[F_HTML]): + + job2['links'].append({ + 'href': job_result_url, + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/results', # noqa + 'type': job_['mimetype'], + 'title': f"results of job {job_id} as {job_['mimetype']}" # noqa + }) + + serialized_jobs['jobs'].append(job2) + + if job_id is None: + j2_template = 'jobs/index.html' + else: + serialized_jobs = serialized_jobs['jobs'][0] + j2_template = 'jobs/job.html' + + if request.format == F_HTML: + data = { + 'jobs': serialized_jobs, + 'now': datetime.now(timezone.utc).strftime(DATETIME_FORMAT) + } + response = render_j2_template(api.tpl_config, j2_template, data, + request.locale) + return headers, HTTPStatus.OK, response + + return headers, HTTPStatus.OK, to_json(serialized_jobs, + api.pretty_print) + + +def execute_process(api: API, request: APIRequest, + process_id) -> Tuple[dict, int, str]: + """ + Execute process + + :param request: A request object + :param process_id: id of process + + :returns: tuple of headers, status code, content + """ + + # Responses are always in US English only + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + if process_id not in api.manager.processes: + msg = 'identifier not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchProcess', msg) + + data = request.data + if not data: + # TODO not all processes require input, e.g. time-dependent or + # random value generators + msg = 'missing request data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'MissingParameterValue', msg) + + try: + # Parse bytes data, if applicable + data = data.decode() + LOGGER.debug(data) + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input does not appear to be valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + data_dict = data.get('inputs', {}) + LOGGER.debug(data_dict) + + subscriber = None + subscriber_dict = data.get('subscriber') + if subscriber_dict: + try: + success_uri = subscriber_dict['successUri'] + except KeyError: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'MissingParameterValue', 'Missing successUri') + else: + subscriber = Subscriber( + # NOTE: successUri is mandatory according to the standard + success_uri=success_uri, + in_progress_uri=subscriber_dict.get('inProgressUri'), + failed_uri=subscriber_dict.get('failedUri'), + ) + + try: + execution_mode = RequestedProcessExecutionMode( + request.headers.get('Prefer', request.headers.get('prefer')) + ) + except ValueError: + execution_mode = None + try: + LOGGER.debug('Executing process') + result = api.manager.execute_process( + process_id, data_dict, execution_mode=execution_mode, + subscriber=subscriber) + job_id, mime_type, outputs, status, additional_headers = result + headers.update(additional_headers or {}) + headers['Location'] = f'{api.base_url}/jobs/{job_id}' + except ProcessorExecuteError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, + request.format, err.ogc_exception_code, err.message) + + response = {} + if status == JobStatus.failed: + response = outputs + + if data.get('response', 'raw') == 'raw': + headers['Content-Type'] = mime_type + response = outputs + elif status not in (JobStatus.failed, JobStatus.accepted): + response['outputs'] = [outputs] + + if status == JobStatus.accepted: + http_status = HTTPStatus.CREATED + elif status == JobStatus.failed: + http_status = HTTPStatus.BAD_REQUEST + else: + http_status = HTTPStatus.OK + + if mime_type == 'application/json': + response2 = to_json(response, api.pretty_print) + else: + response2 = response + + return headers, http_status, response2 + + +def get_job_result(api: API, request: APIRequest, + job_id) -> Tuple[dict, int, str]: + """ + Get result of job (instance of a process) + + :param request: A request object + :param job_id: ID of job + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + try: + job = api.manager.get_job(job_id) + except JobNotFoundError: + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'NoSuchJob', job_id + ) + + status = JobStatus[job['status']] + + if status == JobStatus.running: + msg = 'job still running' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'ResultNotReady', msg) + + elif status == JobStatus.accepted: + # NOTE: this case is not mentioned in the specification + msg = 'job accepted but not yet running' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, + request.format, 'ResultNotReady', msg) + + elif status == JobStatus.failed: + msg = 'job failed' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + + try: + mimetype, job_output = api.manager.get_job_result(job_id) + except JobResultNotFoundError: + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'JobResultNotFound', job_id + ) + + if mimetype not in (None, FORMAT_TYPES[F_JSON]): + headers['Content-Type'] = mimetype + content = job_output + else: + if request.format == F_JSON: + content = json.dumps(job_output, sort_keys=True, indent=4, + default=json_serial) + else: + # HTML + headers['Content-Type'] = "text/html" + data = { + 'job': {'id': job_id}, + 'result': job_output + } + content = render_j2_template( + api.config, 'jobs/results/index.html', + data, request.locale) + + return headers, HTTPStatus.OK, content + + +def delete_job( + api: API, request: APIRequest, job_id +) -> Tuple[dict, int, str]: + """ + Delete a process job + + :param job_id: job identifier + + :returns: tuple of headers, status code, content + """ + response_headers = request.get_response_headers( + SYSTEM_LOCALE, **api.api_headers) + try: + success = api.manager.delete_job(job_id) + except JobNotFoundError: + return api.get_exception( + HTTPStatus.NOT_FOUND, response_headers, request.format, + 'NoSuchJob', job_id + ) + else: + if success: + http_status = HTTPStatus.OK + jobs_url = f"{api.base_url}/jobs" + + response = { + 'jobID': job_id, + 'status': JobStatus.dismissed.value, + 'message': 'Job dismissed', + 'progress': 100, + 'links': [{ + 'href': jobs_url, + 'rel': 'up', + 'type': FORMAT_TYPES[F_JSON], + 'title': 'The job list for the current process' + }] + } + else: + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, response_headers, + request.format, 'InternalError', job_id + ) + LOGGER.info(response) + # TODO: this response does not have any headers + return {}, http_status, response + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML + + LOGGER.debug('setting up processes endpoints') + + oas = {'tags': []} + + paths = {} + + process_manager = get_manager(cfg) + + if len(process_manager.processes) > 0: + paths['/processes'] = { + 'get': { + 'summary': 'Processes', + 'description': 'Processes', + 'tags': ['server'], + 'operationId': 'getProcesses', + 'parameters': [ + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ProcessList.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + LOGGER.debug('setting up processes') + + for k, v in process_manager.processes.items(): + if k.startswith('_'): + LOGGER.debug(f'Skipping hidden layer: {k}') + continue + name = l10n.translate(k, locale) + p = process_manager.get_processor(k) + md_desc = l10n.translate(p.metadata['description'], locale) + process_name_path = f'/processes/{name}' + tag = { + 'name': name, + 'description': md_desc, + 'externalDocs': {} + } + for link in p.metadata.get('links', []): + if link['type'] == 'information': + translated_link = l10n.translate(link, locale) + tag['externalDocs']['description'] = translated_link[ + 'type'] + tag['externalDocs']['url'] = translated_link['url'] + break + if len(tag['externalDocs']) == 0: + del tag['externalDocs'] + + oas['tags'].append(tag) + + paths[process_name_path] = { + 'get': { + 'summary': 'Get process metadata', + 'description': md_desc, + 'tags': [name], + 'operationId': f'describe{name.capitalize()}Process', + 'parameters': [ + {'$ref': '#/components/parameters/f'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + paths[f'{process_name_path}/execution'] = { + 'post': { + 'summary': f"Process {l10n.translate(p.metadata['title'], locale)} execution", # noqa + 'description': md_desc, + 'tags': [name], + 'operationId': f'execute{name.capitalize()}Job', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '201': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ExecuteAsync.yaml"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ServerError.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + }, + 'requestBody': { + 'description': 'Mandatory execute request JSON', + 'required': True, + 'content': { + 'application/json': { + 'schema': { + '$ref': f"{OPENAPI_YAML['oapip']}/schemas/execute.yaml" # noqa + } + } + } + } + } + } + if 'example' in p.metadata: + paths[f'{process_name_path}/execution']['post']['requestBody']['content']['application/json']['example'] = p.metadata['example'] # noqa + + name_in_path = { + 'name': 'jobId', + 'in': 'path', + 'description': 'job identifier', + 'required': True, + 'schema': { + 'type': 'string' + } + } + + paths['/jobs'] = { + 'get': { + 'summary': 'Retrieve jobs list', + 'description': 'Retrieve a list of jobs', + 'tags': ['jobs'], + 'operationId': 'getJobs', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + paths['/jobs/{jobId}'] = { + 'get': { + 'summary': 'Retrieve job details', + 'description': 'Retrieve job details', + 'tags': ['jobs'], + 'parameters': [ + name_in_path, + {'$ref': '#/components/parameters/f'} + ], + 'operationId': 'getJob', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + }, + 'delete': { + 'summary': 'Cancel / delete job', + 'description': 'Cancel / delete job', + 'tags': ['jobs'], + 'parameters': [ + name_in_path + ], + 'operationId': 'deleteJob', + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + }, + } + + paths['/jobs/{jobId}/results'] = { + 'get': { + 'summary': 'Retrieve job results', + 'description': 'Retrive job resiults', + 'tags': ['jobs'], + 'parameters': [ + name_in_path, + {'$ref': '#/components/parameters/f'} + ], + 'operationId': 'getJobResults', + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} + } + } + } + + return [{'name': 'proceses'}, {'name': 'jobs'}], {'paths': paths} diff --git a/pygeoapi/api/stac.py b/pygeoapi/api/stac.py new file mode 100644 index 0000000..948b688 --- /dev/null +++ b/pygeoapi/api/stac.py @@ -0,0 +1,256 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from http import HTTPStatus +import logging +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.plugin import load_plugin + +from pygeoapi.provider.base import ( + ProviderConnectionError, ProviderNotFoundError +) +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_dict_by_key_value, + render_j2_template +) + +from . import APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML + + +LOGGER = logging.getLogger(__name__) + + +CONFORMANCE_CLASSES = [] + + +# TODO: no tests for this? +def get_stac_root(api: API, request: APIRequest) -> Tuple[dict, int, str]: + """ + Provide STAC root page + + :param request: APIRequest instance with query params + + :returns: tuple of headers, status code, content + """ + headers = request.get_response_headers(**api.api_headers) + + id_ = 'pygeoapi-stac' + stac_version = '1.0.0-rc.2' + stac_url = f'{api.base_url}/stac' + + content = { + 'id': id_, + 'type': 'Catalog', + 'stac_version': stac_version, + 'title': l10n.translate( + api.config['metadata']['identification']['title'], + request.locale), + 'description': l10n.translate( + api.config['metadata']['identification']['description'], + request.locale), + 'links': [] + } + + stac_collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'stac-collection') + + for key, value in stac_collections.items(): + content['links'].append({ + 'rel': 'child', + 'href': f'{stac_url}/{key}?f={F_JSON}', + 'type': FORMAT_TYPES[F_JSON] + }) + content['links'].append({ + 'rel': 'child', + 'href': f'{stac_url}/{key}', + 'type': FORMAT_TYPES[F_HTML] + }) + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, + 'stac/collection.html', + content, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + +# TODO: no tests for this? +def get_stac_path(api: API, request: APIRequest, + path) -> Tuple[dict, int, str]: + """ + Provide STAC resource path + + :param request: APIRequest instance with query params + + :returns: tuple of headers, status code, content + """ + headers = request.get_response_headers(**api.api_headers) + + dataset = None + LOGGER.debug(f'Path: {path}') + dir_tokens = path.split('/') + if dir_tokens: + dataset = dir_tokens[0] + + stac_collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'stac-collection') + + if dataset not in stac_collections: + msg = 'Collection not found' + return api.get_exception(HTTPStatus.NOT_FOUND, headers, + request.format, 'NotFound', msg) + + LOGGER.debug('Loading provider') + try: + p = load_plugin('provider', get_provider_by_type( + stac_collections[dataset]['providers'], 'stac')) + except ProviderConnectionError as err: + LOGGER.error(err) + msg = 'connection error (check logs)' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'NoApplicableCode', msg) + + id_ = f'{dataset}-stac' + stac_version = '1.0.0-rc.2' + + content = { + 'id': id_, + 'type': 'Catalog', + 'stac_version': stac_version, + 'description': l10n.translate( + stac_collections[dataset]['description'], request.locale), + 'links': [] + } + try: + stac_data = p.get_data_path( + f'{api.base_url}/stac', + path, + path.replace(dataset, '', 1) + ) + except ProviderNotFoundError as err: + LOGGER.error(err) + msg = 'resource not found' + return api.get_exception(HTTPStatus.NOT_FOUND, headers, + request.format, 'NotFound', msg) + except Exception as err: + LOGGER.error(err) + msg = 'data query error' + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, headers, + request.format, 'NoApplicableCode', msg) + + if isinstance(stac_data, dict): + content.update(stac_data) + content['links'].extend( + stac_collections[dataset].get('links', [])) + + if request.format == F_HTML: # render + content['path'] = path + if 'assets' in content: # item view + if content['type'] == 'Collection': + content = render_j2_template( + api.tpl_config, + 'stac/collection_base.html', + content, + request.locale + ) + elif content['type'] == 'Feature': + content = render_j2_template( + api.tpl_config, + 'stac/item.html', + content, + request.locale + ) + else: + msg = f'Unknown STAC type {content.type}' + LOGGER.error(msg) + return api.get_exception( + HTTPStatus.INTERNAL_SERVER_ERROR, + headers, + request.format, + 'NoApplicableCode', + msg) + else: + content = render_j2_template(api.tpl_config, + 'stac/catalog.html', + content, request.locale) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(content, api.pretty_print) + + else: # send back file + headers.pop('Content-Type', None) + return headers, HTTPStatus.OK, stac_data + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + LOGGER.debug('setting up STAC') + stac_collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'stac-collection') + paths = {} + if stac_collections: + paths['/stac'] = { + 'get': { + 'summary': 'SpatioTemporal Asset Catalog', + 'description': 'SpatioTemporal Asset Catalog', + 'tags': ['stac'], + 'operationId': 'getStacCatalog', + 'parameters': [], + 'responses': { + '200': {'$ref': '#/components/responses/200'}, + 'default': {'$ref': '#/components/responses/default'} + } + } + } + return [{'name': 'stac'}], {'paths': paths} diff --git a/pygeoapi/api/tiles.py b/pygeoapi/api/tiles.py new file mode 100644 index 0000000..5f6dd5f --- /dev/null +++ b/pygeoapi/api/tiles.py @@ -0,0 +1,534 @@ +# ================================================================= + +# Authors: Tom Kralidis +# Francesco Bartoli +# Sander Schaminee +# John A Stevenson +# Colin Blackburn +# Ricardo Garcia Silva +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 Francesco Bartoli +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# Copyright (c) 2023 Ricardo Garcia Silva +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import logging +from http import HTTPStatus +from typing import Tuple + +from pygeoapi import l10n +from pygeoapi.plugin import load_plugin +from pygeoapi.models.provider.base import (TilesMetadataFormat, + TileMatrixSetEnum) +from pygeoapi.provider.base import ( + ProviderGenericError, ProviderTypeError +) + +from pygeoapi.util import ( + get_provider_by_type, to_json, filter_dict_by_key_value, + filter_providers_by_type, render_j2_template +) + +from . import ( + APIRequest, API, FORMAT_TYPES, F_JSON, F_HTML, SYSTEM_LOCALE, F_JSONLD +) + +LOGGER = logging.getLogger(__name__) + +CONFORMANCE_CLASSES = [ + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/mvt', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tileset', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tilesets-list', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/oas30', + 'http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/geodata-tilesets' +] + + +def get_collection_tiles(api: API, request: APIRequest, + dataset=None) -> Tuple[dict, int, str]: + """ + Provide collection tiles + + :param request: A request object + :param dataset: name of collection + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection tiles') + LOGGER.debug('Loading provider') + try: + t = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + except (KeyError, ProviderTypeError): + msg = 'Invalid collection tiles' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + tiles = { + 'links': [], + 'tilesets': [] + } + + tiles['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': 'This document as JSON', + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_JSON}' + }) + tiles['links'].append({ + 'type': FORMAT_TYPES[F_JSONLD], + 'rel': request.get_linkrel(F_JSONLD), + 'title': 'This document as RDF (JSON-LD)', + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_JSONLD}' + }) + tiles['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': 'This document as HTML', + 'href': f'{api.get_collections_url()}/{dataset}/tiles?f={F_HTML}' + }) + + tile_services = p.get_tiles_service( + baseurl=api.base_url, + servicepath=f'{api.get_collections_url()}/{dataset}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f={p.format_type}' # noqa + ) + + for service in tile_services['links']: + tiles['links'].append(service) + + tiling_schemes = p.get_tiling_schemes() + + for matrix in tiling_schemes: + tile_matrix = { + 'title': dataset, + 'tileMatrixSetURI': matrix.tileMatrixSetURI, + 'crs': matrix.crs, + 'dataType': 'vector', + 'links': [] + } + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': 'http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme', + 'title': f'{matrix.tileMatrixSet} TileMatrixSet definition (as {F_JSON})', # noqa + 'href': f'{api.base_url}/TileMatrixSets/{matrix.tileMatrixSet}?f={F_JSON}' # noqa + }) + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_JSON], + 'rel': request.get_linkrel(F_JSON), + 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_JSON}', + 'href': f'{api.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_JSON}' # noqa + }) + tile_matrix['links'].append({ + 'type': FORMAT_TYPES[F_HTML], + 'rel': request.get_linkrel(F_HTML), + 'title': f'{dataset} - {matrix.tileMatrixSet} - {F_HTML}', + 'href': f'{api.get_collections_url()}/{dataset}/tiles/{matrix.tileMatrixSet}?f={F_HTML}' # noqa + }) + + tiles['tilesets'].append(tile_matrix) + + if request.format == F_HTML: # render + tiles['id'] = dataset + tiles['title'] = l10n.translate( + api.config['resources'][dataset]['title'], SYSTEM_LOCALE) + tiles['tilesets'] = [ + scheme.tileMatrixSet for scheme in p.get_tiling_schemes()] + tiles['bounds'] = \ + api.config['resources'][dataset]['extents']['spatial']['bbox'] + tiles['minzoom'] = p.options['zoom']['min'] + tiles['maxzoom'] = p.options['zoom']['max'] + tiles['collections_path'] = api.get_collections_url() + tiles['tile_type'] = p.tile_type + + content = render_j2_template(api.tpl_config, + 'collections/tiles/index.html', tiles, + request.locale) + + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tiles, api.pretty_print) + + +# TODO: no test for this function? +def get_collection_tiles_data( + api: API, request: APIRequest, + dataset=None, matrix_id=None, + z_idx=None, y_idx=None, x_idx=None) -> Tuple[dict, int, str]: + """ + Get collection items tiles + + :param request: A request object + :param dataset: dataset name + :param matrix_id: matrix identifier + :param z_idx: z index + :param y_idx: y index + :param x_idx: x index + + :returns: tuple of headers, status code, content + """ + + format_ = request.format + if not format_: + return api.get_format_exception(request) + headers = request.get_response_headers(SYSTEM_LOCALE, + **api.api_headers) + LOGGER.debug('Processing tiles') + + collections = filter_dict_by_key_value(api.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Loading tile provider') + try: + t = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + + format_ = p.format_type + headers['Content-Type'] = format_ + + LOGGER.debug(f'Fetching tileset id {matrix_id} and tile {z_idx}/{y_idx}/{x_idx}') # noqa + content = p.get_tiles(layer=p.get_layer(), tileset=matrix_id, + z=z_idx, y=y_idx, x=x_idx, format_=format_) + if content is None: + msg = 'identifier not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, format_, 'NotFound', msg) + else: + return headers, HTTPStatus.OK, content + + # @TODO: figure out if the spec requires to return json errors + except KeyError: + msg = 'Invalid collection tiles' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, format_, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + +# TODO: no test for this function? +def get_collection_tiles_metadata( + api: API, request: APIRequest, + dataset=None, matrix_id=None) -> Tuple[dict, int, str]: + """ + Get collection items tiles + + :param request: A request object + :param dataset: dataset name + :param matrix_id: matrix identifier + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid([TilesMetadataFormat.TILEJSON]): + return api.get_format_exception(request) + headers = request.get_response_headers(**api.api_headers) + + if any([dataset is None, + dataset not in api.config['resources'].keys()]): + + msg = 'Collection not found' + return api.get_exception( + HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg) + + LOGGER.debug('Creating collection tiles') + LOGGER.debug('Loading provider') + try: + t = get_provider_by_type( + api.config['resources'][dataset]['providers'], 'tile') + p = load_plugin('provider', t) + except KeyError: + msg = 'Invalid collection tiles' + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', msg) + except ProviderGenericError as err: + LOGGER.error(err) + return api.get_exception( + err.http_status_code, headers, request.format, + err.ogc_exception_code, err.message) + + # Get provider language (if any) + prv_locale = l10n.get_plugin_locale(t, request.raw_locale) + + # Set response language to requested provider locale + # (if it supports language) and/or otherwise the requested pygeoapi + # locale (or fallback default locale) + l10n.set_response_language(headers, prv_locale, request.locale) + + tiles_metadata = p.get_metadata( + dataset=dataset, server_url=api.base_url, + layer=p.get_layer(), tileset=matrix_id, + metadata_format=request._format, title=l10n.translate( + api.config['resources'][dataset]['title'], + request.locale), + description=l10n.translate( + api.config['resources'][dataset]['description'], + request.locale), + language=prv_locale) + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, + 'collections/tiles/metadata.html', + tiles_metadata, request.locale) + + return headers, HTTPStatus.OK, content + else: + return headers, HTTPStatus.OK, tiles_metadata + + +def tilematrixsets(api: API, + request: APIRequest) -> Tuple[dict, int, str]: + """ + Provide tileMatrixSets definition + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**api.api_headers) + + # Retrieve available TileMatrixSets + enums = [e.value for e in TileMatrixSetEnum] + + tms = {"tileMatrixSets": []} + + for e in enums: + tms['tileMatrixSets'].append({ + "title": e.title, + "id": e.tileMatrixSet, + "uri": e.tileMatrixSetURI, + "links": [ + { + "rel": "self", + "type": "text/html", + "title": f"The HTML representation of the {e.tileMatrixSet} tile matrix set", # noqa + "href": f"{api.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=html" # noqa + }, + { + "rel": "self", + "type": "application/json", + "title": f"The JSON representation of the {e.tileMatrixSet} tile matrix set", # noqa + "href": f"{api.base_url}/TileMatrixSets/{e.tileMatrixSet}?f=json" # noqa + } + ] + }) + + tms['links'] = [{ + "rel": "alternate", + "type": "text/html", + "title": "This document as HTML", + "href": f"{api.base_url}/tileMatrixSets?f=html" + }, { + "rel": "self", + "type": "application/json", + "title": "This document", + "href": f"{api.base_url}/tileMatrixSets?f=json" + }] + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, + 'tilematrixsets/index.html', + tms, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tms, api.pretty_print) + + +def tilematrixset(api: API, + request: APIRequest, + tileMatrixSetId) -> Tuple[dict, + int, str]: + """ + Provide tile matrix definition + + :param request: A request object + + :returns: tuple of headers, status code, content + """ + + headers = request.get_response_headers(**api.api_headers) + + # Retrieve relevant TileMatrixSet + enums = [e.value for e in TileMatrixSetEnum] + enum = None + + try: + for e in enums: + if tileMatrixSetId == e.tileMatrixSet: + enum = e + if not enum: + raise ValueError('could not find this tilematrixset') + except ValueError as err: + return api.get_exception( + HTTPStatus.BAD_REQUEST, headers, request.format, + 'InvalidParameterValue', str(err)) + + tms = { + "title": enum.tileMatrixSet, + "crs": enum.crs, + "id": enum.tileMatrixSet, + "uri": enum.tileMatrixSetURI, + "orderedAxes": enum.orderedAxes, + "wellKnownScaleSet": enum.wellKnownScaleSet, + "tileMatrices": enum.tileMatrices + } + + if request.format == F_HTML: # render + content = render_j2_template(api.tpl_config, + 'tilematrixsets/tilematrixset.html', + tms, request.locale) + return headers, HTTPStatus.OK, content + + return headers, HTTPStatus.OK, to_json(tms, api.pretty_print) + + +def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]: # noqa + """ + Get OpenAPI fragments + + :param cfg: `dict` of configuration + :param locale: `str` of locale + + :returns: `tuple` of `list` of tag objects, and `dict` of path objects + """ + + from pygeoapi.openapi import OPENAPI_YAML, get_visible_collections + + paths = {} + + LOGGER.debug('setting up tiles endpoints') + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + + for k, v in get_visible_collections(cfg).items(): + tile_extension = filter_providers_by_type( + collections[k]['providers'], 'tile') + + if tile_extension: + tp = load_plugin('provider', tile_extension) + + tiles_path = f'/collections/{k}/tiles' + title = l10n.translate(v['title'], locale) + description = l10n.translate(v['description'], locale) + + paths[tiles_path] = { + 'get': { + 'summary': f'Fetch a {title} tiles description', + 'description': description, + 'tags': [k], + 'operationId': f'describe{k.capitalize()}Tiles', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': '#/components/responses/Tiles'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + + tiles_data_path = f'{tiles_path}/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}' # noqa + + paths[tiles_data_path] = { + 'get': { + 'summary': f'Get a {title} tile', + 'description': description, + 'tags': [k], + 'operationId': f'get{k.capitalize()}Tiles', + 'parameters': [ + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrixSetId"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrix"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileRow"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileCol"}, # noqa + { + 'name': 'f', + 'in': 'query', + 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document.', # noqa + 'required': False, + 'schema': { + 'type': 'string', + 'enum': [tp.format_type], + 'default': tp.format_type + }, + 'style': 'form', + 'explode': False + } + ], + 'responses': { + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + mimetype = tile_extension['format']['mimetype'] + paths[tiles_data_path]['get']['responses']['200'] = { + 'description': 'successful operation', + 'content': { + mimetype: { + 'schema': { + 'type': 'string', + 'format': 'binary' + } + } + } + } + + return [{'name': 'tiles'}], {'paths': paths} diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index 974828b..fe841b2 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -8,7 +8,7 @@ # Copyright (c) 2022 Francesco Bartoli # Copyright (c) 2022 Luca Delucchi # Copyright (c) 2022 Krishna Lodha -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -40,7 +40,14 @@ from typing import Tuple, Dict, Mapping, Optional from django.conf import settings from django.http import HttpRequest, HttpResponse -from pygeoapi.api import API +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.coverages as coverages_api +import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes as itemtypes_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api def landing_page(request: HttpRequest) -> HttpResponse: @@ -157,8 +164,8 @@ def collection_queryables(request: HttpRequest, :returns: Django HTTP Response """ - response_ = _feed_response( - request, 'get_collection_queryables', collection_id + response_ = execute_from_django( + itemtypes_api.get_collection_queryables, request, collection_id ) response = _to_django_response(*response_) @@ -176,22 +183,26 @@ def collection_items(request: HttpRequest, collection_id: str) -> HttpResponse: """ if request.method == 'GET': - response_ = _feed_response( + response_ = execute_from_django( + itemtypes_api.get_collection_items, request, - 'get_collection_items', collection_id, + skip_valid_check=True, ) elif request.method == 'POST': if request.content_type is not None: if request.content_type == 'application/geo+json': - response_ = _feed_response(request, 'manage_collection_item', - request, 'create', collection_id) + response_ = execute_from_django( + itemtypes_api.manage_collection_item, request, + 'create', collection_id, skip_valid_check=True) else: - response_ = _feed_response(request, 'post_collection_items', - request, collection_id) + response_ = execute_from_django( + itemtypes_api.post_collection_items, + request, collection_id, skip_valid_check=True,) elif request.method == 'OPTIONS': - response_ = _feed_response(request, 'manage_collection_item', - request, 'options', collection_id) + response_ = execute_from_django(itemtypes_api.manage_collection_item, + request, 'options', collection_id, + skip_valid_check=True) response = _to_django_response(*response_) @@ -206,12 +217,9 @@ def collection_map(request: HttpRequest, collection_id: str): :returns: HTTP response """ - - response_ = _feed_response(request, 'get_collection_map', collection_id) - - response = _to_django_response(*response_) - - return response + return execute_from_django( + maps_api.get_collection_map, request, collection_id + ) def collection_style_map(request: HttpRequest, collection_id: str, @@ -252,17 +260,17 @@ def collection_item(request: HttpRequest, elif request.method == 'PUT': response_ = _feed_response( request, 'manage_collection_item', request, 'update', - collection_id, item_id + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'DELETE': response_ = _feed_response( request, 'manage_collection_item', request, 'delete', - collection_id, item_id + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'OPTIONS': response_ = _feed_response( request, 'manage_collection_item', request, 'options', - collection_id, item_id) + collection_id, item_id, skip_valid_check=True) response = _to_django_response(*response_) @@ -280,8 +288,8 @@ def collection_coverage(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response( - request, 'get_collection_coverage', collection_id + response_ = execute_from_django( + coverages_api.get_collection_coverage, request, collection_id ) response = _to_django_response(*response_) @@ -298,10 +306,8 @@ def collection_tiles(request: HttpRequest, collection_id: str) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_collection_tiles', collection_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(tiles_api.get_collection_tiles, request, + collection_id) def collection_tiles_metadata(request: HttpRequest, collection_id: str, @@ -316,15 +322,11 @@ def collection_tiles_metadata(request: HttpRequest, collection_id: str, :returns: Django HTTP response """ - response_ = _feed_response( - request, - 'get_collection_tiles_metadata', - collection_id, - tileMatrixSetId, + return execute_from_django( + tiles_api.get_collection_tiles_metadata, + request, collection_id, tileMatrixSetId, + skip_valid_check=True, ) - response = _to_django_response(*response_) - - return response def collection_item_tiles(request: HttpRequest, collection_id: str, @@ -343,18 +345,16 @@ def collection_item_tiles(request: HttpRequest, collection_id: str, :returns: Django HTTP response """ - response_ = _feed_response( + return execute_from_django( + tiles_api.get_collection_tiles_data, request, - 'get_collection_tiles_data', collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol, + skip_valid_check=True, ) - response = _to_django_response(*response_) - - return response def processes(request: HttpRequest, @@ -368,10 +368,8 @@ def processes(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response(request, 'describe_processes', process_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(processes_api.describe_processes, request, + process_id) def jobs(request: HttpRequest, job_id: Optional[str] = None) -> HttpResponse: @@ -384,11 +382,7 @@ def jobs(request: HttpRequest, job_id: Optional[str] = None) -> HttpResponse: :returns: Django HTTP response """ - - response_ = _feed_response(request, 'get_jobs', job_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(processes_api.get_jobs, request, job_id) def job_results(request: HttpRequest, @@ -402,10 +396,7 @@ def job_results(request: HttpRequest, :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_job_result', job_id) - response = _to_django_response(*response_) - - return response + return execute_from_django(processes_api.get_job_result, request, job_id) def job_results_resource(request: HttpRequest, process_id: str, job_id: str, @@ -420,6 +411,7 @@ def job_results_resource(request: HttpRequest, process_id: str, job_id: str, :returns: Django HTTP response """ + # TODO: this api method does not exist response_ = _feed_response( request, 'get_job_result_resource', @@ -451,17 +443,15 @@ def get_collection_edr_query( query_type = 'locations' else: query_type = request.path.split('/')[-1] - response_ = _feed_response( + return execute_from_django( + edr_api.get_collection_edr_query, request, - 'get_collection_edr_query', collection_id, instance_id, query_type, - location_id + location_id, + skip_valid_check=True, ) - response = _to_django_response(*response_) - - return response def stac_catalog_root(request: HttpRequest) -> HttpResponse: @@ -473,10 +463,7 @@ def stac_catalog_root(request: HttpRequest) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_stac_root') - response = _to_django_response(*response_) - - return response + return execute_from_django(stac_api.get_stac_root, request) def stac_catalog_path(request: HttpRequest, path: str) -> HttpResponse: @@ -489,10 +476,7 @@ def stac_catalog_path(request: HttpRequest, path: str) -> HttpResponse: :returns: Django HTTP response """ - response_ = _feed_response(request, 'get_stac_path', path) - response = _to_django_response(*response_) - - return response + return execute_from_django(stac_api.get_stac_path, request, path) def admin_config(request: HttpRequest) -> HttpResponse: @@ -551,6 +535,7 @@ def admin_config_resource(request: HttpRequest, resource_id) +# TODO: remove this when all views have been refactored def _feed_response(request: HttpRequest, api_definition: str, *args, **kwargs) -> Tuple[Dict, int, str]: """Use pygeoapi api to process the input request""" @@ -566,8 +551,31 @@ def _feed_response(request: HttpRequest, api_definition: str, return api(request, *args, **kwargs) +def execute_from_django(api_function, request: HttpRequest, *args, + skip_valid_check=False) -> HttpResponse: + + api_: API | "Admin" + if settings.PYGEOAPI_CONFIG['server'].get('admin'): # noqa + from pygeoapi.admin import Admin + api_ = Admin(settings.PYGEOAPI_CONFIG, settings.OPENAPI_DOCUMENT) + else: + api_ = API(settings.PYGEOAPI_CONFIG, settings.OPENAPI_DOCUMENT) + + api_request = APIRequest.from_django(request, api_.locales) + content: str | bytes + if not skip_valid_check and not api_request.is_valid(): + headers, status, content = api_.get_format_exception(api_request) + else: + + headers, status, content = api_function(api_, api_request, *args) + content = apply_gzip(headers, content) + + return _to_django_response(headers, status, content) + + +# TODO: inline this to execute_from_django after refactoring def _to_django_response(headers: Mapping, status_code: int, - content: str) -> HttpResponse: + content: str | bytes) -> HttpResponse: """Convert API payload to a django response""" response = HttpResponse(content, status=status_code) diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 5c22475..e4d8e82 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -34,9 +34,17 @@ import os import click -from flask import Flask, Blueprint, make_response, request, send_from_directory +from flask import (Flask, Blueprint, make_response, request, + send_from_directory, Response, Request) -from pygeoapi.api import API +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.coverages as coverages_api +import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes as itemtypes_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_mimetype, get_api_rules @@ -110,6 +118,7 @@ if (OGC_SCHEMAS_LOCATION is not None and mimetype=get_mimetype(basename_)) +# TODO: inline in execute_from_flask when all views have been refactored def get_response(result: tuple): """ Creates a Flask Response object and updates matching headers. @@ -117,7 +126,7 @@ def get_response(result: tuple): :param result: The result of the API call. This should be a tuple of (headers, status, content). - :returns: A Response instance. + :returns: A Response instance """ headers, status, content = result @@ -128,6 +137,33 @@ def get_response(result: tuple): return response +def execute_from_flask(api_function, request: Request, *args, + skip_valid_check=False) -> Response: + """ + Executes API function from Flask + + :param api_function: API function + :param request: request object + :param *args: variable length additional arguments + :param skip_validity_check: bool + + :returns: A Response instance + """ + + api_request = APIRequest.from_flask(request, api_.locales) + + content: str | bytes + + if not skip_valid_check and not api_request.is_valid(): + headers, status, content = api_.get_format_exception(api_request) + else: + headers, status, content = api_function(api_, api_request, *args) + content = apply_gzip(headers, content) + # handle jsonld too? + + return get_response((headers, status, content)) + + @BLUEPRINT.route('/') def landing_page(): """ @@ -145,6 +181,7 @@ def openapi(): :returns: HTTP response """ + return get_response(api_.openapi_(request)) @@ -155,6 +192,7 @@ def conformance(): :returns: HTTP response """ + return get_response(api_.conformance(request)) @@ -164,9 +202,12 @@ def get_tilematrix_set(tileMatrixSetId=None): OGC API TileMatrixSet endpoint :param tileMatrixSetId: identifier of tile matrix set + :returns: HTTP response """ - return get_response(api_.tilematrixset(request, tileMatrixSetId)) + + return execute_from_flask(tiles_api.tilematrixset, request, + tileMatrixSetId) @BLUEPRINT.route('/TileMatrixSets') @@ -176,7 +217,8 @@ def get_tilematrix_sets(): :returns: HTTP response """ - return get_response(api_.tilematrixsets(request)) + + return execute_from_flask(tiles_api.tilematrixsets, request) @BLUEPRINT.route('/collections') @@ -189,6 +231,7 @@ def collections(collection_id=None): :returns: HTTP response """ + return get_response(api_.describe_collections(request, collection_id)) @@ -201,6 +244,7 @@ def collection_schema(collection_id): :returns: HTTP response """ + return get_response(api_.get_collection_schema(request, collection_id)) @@ -213,7 +257,9 @@ def collection_queryables(collection_id=None): :returns: HTTP response """ - return get_response(api_.get_collection_queryables(request, collection_id)) + + return execute_from_flask(itemtypes_api.get_collection_queryables, request, + collection_id) @BLUEPRINT.route('/collections//items', @@ -234,36 +280,40 @@ def collection_items(collection_id, item_id=None): if item_id is None: if request.method == 'GET': # list items - return get_response( - api_.get_collection_items(request, collection_id)) + return execute_from_flask(itemtypes_api.get_collection_items, + request, collection_id, + skip_valid_check=True) elif request.method == 'POST': # filter or manage items if request.content_type is not None: if request.content_type == 'application/geo+json': - return get_response( - api_.manage_collection_item(request, 'create', - collection_id)) + return execute_from_flask( + itemtypes_api.manage_collection_item, + request, 'create', collection_id, + skip_valid_check=True) else: - return get_response( - api_.post_collection_items(request, collection_id)) + return execute_from_flask( + itemtypes_api.post_collection_items, request, + collection_id, skip_valid_check=True) elif request.method == 'OPTIONS': - return get_response( - api_.manage_collection_item(request, 'options', collection_id)) + return execute_from_flask( + itemtypes_api.manage_collection_item, request, 'options', + collection_id, skip_valid_check=True) elif request.method == 'DELETE': - return get_response( - api_.manage_collection_item(request, 'delete', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'delete', collection_id, item_id, + skip_valid_check=True) elif request.method == 'PUT': - return get_response( - api_.manage_collection_item(request, 'update', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'update', collection_id, item_id, + skip_valid_check=True) elif request.method == 'OPTIONS': - return get_response( - api_.manage_collection_item(request, 'options', - collection_id, item_id)) + return execute_from_flask(itemtypes_api.manage_collection_item, + request, 'options', collection_id, item_id, + skip_valid_check=True) else: - return get_response( - api_.get_collection_item(request, collection_id, item_id)) + return execute_from_flask(itemtypes_api.get_collection_item, request, + collection_id, item_id) @BLUEPRINT.route('/collections//coverage') @@ -275,7 +325,9 @@ def collection_coverage(collection_id): :returns: HTTP response """ - return get_response(api_.get_collection_coverage(request, collection_id)) + + return execute_from_flask(coverages_api.get_collection_coverage, request, + collection_id) @BLUEPRINT.route('/collections//tiles') @@ -287,8 +339,9 @@ def get_collection_tiles(collection_id=None): :returns: HTTP response """ - return get_response(api_.get_collection_tiles( - request, collection_id)) + + return execute_from_flask(tiles_api.get_collection_tiles, request, + collection_id) @BLUEPRINT.route('/collections//tiles/') @@ -302,8 +355,10 @@ def get_collection_tiles_metadata(collection_id=None, tileMatrixSetId=None): :returns: HTTP response """ - return get_response(api_.get_collection_tiles_metadata( - request, collection_id, tileMatrixSetId)) + + return execute_from_flask(tiles_api.get_collection_tiles_metadata, + request, collection_id, tileMatrixSetId, + skip_valid_check=True) @BLUEPRINT.route('/collections//tiles/\ @@ -321,8 +376,12 @@ def get_collection_tiles_data(collection_id=None, tileMatrixSetId=None, :returns: HTTP response """ - return get_response(api_.get_collection_tiles_data( - request, collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol)) + + return execute_from_flask( + tiles_api.get_collection_tiles_data, + request, collection_id, tileMatrixSetId, tileMatrix, tileRow, tileCol, + skip_valid_check=True, + ) @BLUEPRINT.route('/collections//map') @@ -337,15 +396,9 @@ def collection_map(collection_id, style_id=None): :returns: HTTP response """ - headers, status_code, content = api_.get_collection_map( - request, collection_id, style_id) - - response = make_response(content, status_code) - - if headers: - response.headers = headers - - return response + return execute_from_flask( + maps_api.get_collection_map, request, collection_id, style_id + ) @BLUEPRINT.route('/processes') @@ -358,7 +411,9 @@ def get_processes(process_id=None): :returns: HTTP response """ - return get_response(api_.describe_processes(request, process_id)) + + return execute_from_flask(processes_api.describe_processes, request, + process_id) @BLUEPRINT.route('/jobs') @@ -374,12 +429,12 @@ def get_jobs(job_id=None): """ if job_id is None: - return get_response(api_.get_jobs(request)) + return execute_from_flask(processes_api.get_jobs, request) else: if request.method == 'DELETE': # dismiss job - return get_response(api_.delete_job(request, job_id)) + return execute_from_flask(processes_api.delete_jobs, request) else: # Return status of a specific job - return get_response(api_.get_jobs(request, job_id)) + return execute_from_flask(processes_api.get_jobs, request, job_id) @BLUEPRINT.route('/processes//execution', methods=['POST']) @@ -392,7 +447,8 @@ def execute_process_jobs(process_id): :returns: HTTP response """ - return get_response(api_.execute_process(request, process_id)) + return execute_from_flask(processes_api.execute_process, request, + process_id) @BLUEPRINT.route('/jobs//results', @@ -405,7 +461,8 @@ def get_job_result(job_id=None): :returns: HTTP response """ - return get_response(api_.get_job_result(request, job_id)) + + return execute_from_flask(processes_api.get_job_result, request, job_id) @BLUEPRINT.route('/jobs//results/', @@ -419,6 +476,8 @@ def get_job_result_resource(job_id, resource): :returns: HTTP response """ + + # TODO: this does not seem to exist? return get_response(api_.get_job_result_resource( request, job_id, resource)) @@ -450,14 +509,17 @@ def get_collection_edr_query(collection_id, instance_id=None, :returns: HTTP response """ + if location_id: query_type = 'locations' else: query_type = request.path.split('/')[-1] - return get_response(api_.get_collection_edr_query(request, collection_id, - instance_id, query_type, - location_id)) + return execute_from_flask( + edr_api.get_collection_edr_query, request, collection_id, instance_id, + query_type, location_id, + skip_valid_check=True, + ) @BLUEPRINT.route('/stac') @@ -467,7 +529,8 @@ def stac_catalog_root(): :returns: HTTP response """ - return get_response(api_.get_stac_root(request)) + + return execute_from_flask(stac_api.get_stac_root, request) @BLUEPRINT.route('/stac/') @@ -479,7 +542,8 @@ def stac_catalog_path(path): :returns: HTTP response """ - return get_response(api_.get_stac_path(request, path)) + + return execute_from_flask(stac_api.get_stac_path, request, path) @ADMIN_BLUEPRINT.route('/admin/config', methods=['GET', 'PUT', 'PATCH']) diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index a68c37c..274e19d 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -37,19 +37,16 @@ import json import logging import os from pathlib import Path -from typing import Tuple, Union +from typing import Union import click from jsonschema import validate as jsonschema_validate import yaml from pygeoapi import l10n +from pygeoapi.api import all_apis from pygeoapi.models.openapi import OAPIFormat -from pygeoapi.plugin import load_plugin -from pygeoapi.process.manager.base import get_manager -from pygeoapi.provider.base import ProviderTypeError, SchemaType -from pygeoapi.util import (filter_dict_by_key_value, get_provider_by_type, - filter_providers_by_type, to_json, yaml_load, +from pygeoapi.util import (filter_dict_by_key_value, to_json, yaml_load, get_api_rules, get_base_url) LOGGER = logging.getLogger(__name__) @@ -151,8 +148,7 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: paths = {} # TODO: make openapi multilingual (default language only for now) - server_locales = l10n.get_locales(cfg) - locale_ = server_locales[0] + locale_ = l10n.get_locales(cfg)[0] api_rules = get_api_rules(cfg) @@ -282,11 +278,6 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: 'url': cfg['metadata']['identification']['url']} } ) - oas['tags'].append({ - 'name': 'stac', - 'description': 'SpatioTemporal Asset Catalog' - } - ) oas['components'] = { 'responses': { @@ -307,19 +298,183 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: 'schema': {'$ref': '#/components/schemas/queryables'} } } + } + }, + 'parameters': get_oas_30_parameters(cfg=cfg, locale_=locale_), + 'schemas': { + # TODO: change this schema once OGC will definitively publish it + 'queryable': { + 'type': 'object', + 'required': [ + 'queryable', + 'type' + ], + 'properties': { + 'queryable': { + 'description': 'the token that may be used in a CQL predicate', # noqa + 'type': 'string' + }, + 'title': { + 'description': 'a human readable title for the queryable', # noqa + 'type': 'string' + }, + 'description': { + 'description': 'a human-readable narrative describing the queryable', # noqa + 'type': 'string' + }, + 'language': { + 'description': 'the language used for the title and description', # noqa + 'type': 'string', + 'default': [ + 'en' + ] + }, + 'type': { + 'description': 'the data type of the queryable', # noqa + 'type': 'string' + }, + 'type-ref': { + 'description': 'a reference to the formal definition of the type', # noqa + 'type': 'string', + 'format': 'url' + } + } }, - 'Tiles': { - 'description': 'Retrieves the tiles description for this collection', # noqa - 'content': { - 'application/json': { - 'schema': { - '$ref': '#/components/schemas/tiles' + 'queryables': { + 'type': 'object', + 'required': [ + 'queryables' + ], + 'properties': { + 'queryables': { + 'type': 'array', + 'items': {'$ref': '#/components/schemas/queryable'} + } + } + } + } + } + + items_f = deepcopy(oas['components']['parameters']['f']) + items_f['schema']['enum'].append('csv') + + LOGGER.debug('setting up datasets') + + for k, v in get_visible_collections(cfg).items(): + name = l10n.translate(k, locale_) + title = l10n.translate(v['title'], locale_) + desc = l10n.translate(v['description'], locale_) + collection_name_path = f'/collections/{k}' + tag = { + 'name': name, + 'description': desc, + 'externalDocs': {} + } + for link in l10n.translate(v.get('links', []), locale_): + if link['type'] == 'information': + tag['externalDocs']['description'] = link['type'] + tag['externalDocs']['url'] = link['url'] + break + if len(tag['externalDocs']) == 0: + del tag['externalDocs'] + + oas['tags'].append(tag) + + paths[collection_name_path] = { + 'get': { + 'summary': f'Get {title} metadata', + 'description': desc, + 'tags': [name], + 'operationId': f'describe{name.capitalize()}Collection', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Collection"}, # noqa + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + + oas['components']['responses'].update({ + 'Tiles': { + 'description': 'Retrieves the tiles description for this collection', # noqa + 'content': { + 'application/json': { + 'schema': { + '$ref': '#/components/schemas/tiles' + } } } } } - }, - 'parameters': { + ) + + oas['components']['schemas'].update({ + 'tilematrixsetlink': { + 'type': 'object', + 'required': ['tileMatrixSet'], + 'properties': { + 'tileMatrixSet': { + 'type': 'string' + }, + 'tileMatrixSetURI': { + 'type': 'string' + } + } + }, + 'tiles': { + 'type': 'object', + 'required': [ + 'tileMatrixSetLinks', + 'links' + ], + 'properties': { + 'tileMatrixSetLinks': { + 'type': 'array', + 'items': { + '$ref': '#/components/schemas/tilematrixsetlink' # noqa + } + }, + 'links': { + 'type': 'array', + 'items': {'$ref': f"{OPENAPI_YAML['oapit']}#/components/schemas/link"} # noqa + } + } + } + } + ) + + oas['paths'] = paths + + for api_name, api_module in all_apis().items(): + LOGGER.debug(f'Adding OpenAPI definitions for {api_name}') + + try: + sub_tags, sub_paths = api_module.get_oas_30(cfg, locale_) + oas['paths'].update(sub_paths['paths']) + oas['tags'].extend(sub_tags) + except Exception as err: + if fail_on_invalid_collection: + raise + else: + LOGGER.warning(f'Resource not added to OpenAPI: {err}') + + if cfg['server'].get('admin', False): + schema_dict = get_config_schema() + oas['definitions'] = schema_dict['definitions'] + LOGGER.debug('Adding admin endpoints') + oas['paths'].update(get_admin()) + + return oas + + +def get_oas_30_parameters(cfg: dict, locale_: str): + server_locales = l10n.get_locales(cfg) + return { 'f': { 'name': 'f', 'in': 'query', @@ -344,20 +499,6 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: 'default': l10n.locale2str(locale_) } }, - 'properties': { - 'name': 'properties', - 'in': 'query', - 'description': 'The properties that should be included for each feature. The parameter value is a comma-separated list of property names.', # noqa - 'required': False, - 'style': 'form', - 'explode': False, - 'schema': { - 'type': 'array', - 'items': { - 'type': 'string' - } - } - }, 'skipGeometry': { 'name': 'skipGeometry', 'in': 'query', @@ -461,315 +602,21 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict: 'type': 'string' } } - }, - 'schemas': { - # TODO: change this schema once OGC will definitively publish it - 'queryable': { - 'type': 'object', - 'required': [ - 'queryable', - 'type' - ], - 'properties': { - 'queryable': { - 'description': 'the token that may be used in a CQL predicate', # noqa - 'type': 'string' - }, - 'title': { - 'description': 'a human readable title for the queryable', # noqa - 'type': 'string' - }, - 'description': { - 'description': 'a human-readable narrative describing the queryable', # noqa - 'type': 'string' - }, - 'language': { - 'description': 'the language used for the title and description', # noqa - 'type': 'string', - 'default': [ - 'en' - ] - }, - 'type': { - 'description': 'the data type of the queryable', # noqa - 'type': 'string' - }, - 'type-ref': { - 'description': 'a reference to the formal definition of the type', # noqa - 'type': 'string', - 'format': 'url' - } - } - }, - 'queryables': { - 'type': 'object', - 'required': [ - 'queryables' - ], - 'properties': { - 'queryables': { - 'type': 'array', - 'items': {'$ref': '#/components/schemas/queryable'} - } - } - }, - 'tilematrixsetlink': { - 'type': 'object', - 'required': ['tileMatrixSet'], - 'properties': { - 'tileMatrixSet': { - 'type': 'string' - }, - 'tileMatrixSetURI': { - 'type': 'string' - } - } - }, - 'tiles': { - 'type': 'object', - 'required': [ - 'tileMatrixSetLinks', - 'links' - ], - 'properties': { - 'tileMatrixSetLinks': { - 'type': 'array', - 'items': { - '$ref': '#/components/schemas/tilematrixsetlink' # noqa - } - }, - 'links': { - 'type': 'array', - 'items': {'$ref': f"{OPENAPI_YAML['oapit']}#/components/schemas/link"} # noqa - } - } - } } - } - LOGGER.debug('setting up datasets') + +def get_visible_collections(cfg: dict) -> dict: collections = filter_dict_by_key_value(cfg['resources'], 'type', 'collection') - for k, v in collections.items(): - try: - LOGGER.debug(f'Generating OpenAPI tags/paths for collection {k}') - r_tags, r_paths = handle_collection(locale_, oas['components'], - k, v) - oas['tags'].extend(r_tags) - paths.update(r_paths) - except Exception as err: - if fail_on_invalid_collection: - raise - else: - LOGGER.warning(f'Resource {k} not added to OpenAPI: {err}') - - LOGGER.debug('setting up STAC') - stac_collections = filter_dict_by_key_value(cfg['resources'], - 'type', 'stac-collection') - if stac_collections: - paths['/stac'] = { - 'get': { - 'summary': 'SpatioTemporal Asset Catalog', - 'description': 'SpatioTemporal Asset Catalog', - 'tags': ['stac'], - 'operationId': 'getStacCatalog', - 'parameters': [], - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - 'default': {'$ref': '#/components/responses/default'} - } - } - } - - process_manager = get_manager(cfg) - - if len(process_manager.processes) > 0: - paths['/processes'] = { - 'get': { - 'summary': 'Processes', - 'description': 'Processes', - 'tags': ['server'], - 'operationId': 'getProcesses', - 'parameters': [ - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ProcessList.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - } - } - } - LOGGER.debug('setting up processes') - - for k, v in process_manager.processes.items(): - if k.startswith('_'): - LOGGER.debug(f'Skipping hidden layer: {k}') - continue - name = l10n.translate(k, locale_) - p = process_manager.get_processor(k) - md_desc = l10n.translate(p.metadata['description'], locale_) - process_name_path = f'/processes/{name}' - tag = { - 'name': name, - 'description': md_desc, # noqa - 'externalDocs': {} - } - for link in p.metadata.get('links', []): - if link['type'] == 'information': - translated_link = l10n.translate(link, locale_) - tag['externalDocs']['description'] = translated_link[ - 'type'] - tag['externalDocs']['url'] = translated_link['url'] - break - if len(tag['externalDocs']) == 0: - del tag['externalDocs'] - - oas['tags'].append(tag) - - paths[process_name_path] = { - 'get': { - 'summary': 'Get process metadata', - 'description': md_desc, - 'tags': [name], - 'operationId': f'describe{name.capitalize()}Process', - 'parameters': [ - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - 'default': {'$ref': '#/components/responses/default'} - } - } - } - - paths[f'{process_name_path}/execution'] = { - 'post': { - 'summary': f"Process {l10n.translate(p.metadata['title'], locale_)} execution", # noqa - 'description': md_desc, - 'tags': [name], - 'operationId': f'execute{name.capitalize()}Job', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '201': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ExecuteAsync.yaml"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/ServerError.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - }, - 'requestBody': { - 'description': 'Mandatory execute request JSON', - 'required': True, - 'content': { - 'application/json': { - 'schema': { - '$ref': f"{OPENAPI_YAML['oapip']}/schemas/execute.yaml" # noqa - } - } - } - } - } - } - if 'example' in p.metadata: - paths[f'{process_name_path}/execution']['post']['requestBody']['content']['application/json']['example'] = p.metadata['example'] # noqa - - name_in_path = { - 'name': 'jobId', - 'in': 'path', - 'description': 'job identifier', - 'required': True, - 'schema': { - 'type': 'string' - } - } - - paths['/jobs'] = { - 'get': { - 'summary': 'Retrieve jobs list', - 'description': 'Retrieve a list of jobs', - 'tags': ['jobs'], - 'operationId': 'getJobs', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} - } - } - } - - paths['/jobs/{jobId}'] = { - 'get': { - 'summary': 'Retrieve job details', - 'description': 'Retrieve job details', - 'tags': ['jobs'], - 'parameters': [ - name_in_path, - {'$ref': '#/components/parameters/f'} - ], - 'operationId': 'getJob', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - }, - 'delete': { - 'summary': 'Cancel / delete job', - 'description': 'Cancel / delete job', - 'tags': ['jobs'], - 'parameters': [ - name_in_path - ], - 'operationId': 'deleteJob', - 'responses': { - '204': {'$ref': '#/components/responses/204'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - }, - } - - paths['/jobs/{jobId}/results'] = { - 'get': { - 'summary': 'Retrieve job results', - 'description': 'Retrive job resiults', - 'tags': ['jobs'], - 'parameters': [ - name_in_path, - {'$ref': '#/components/parameters/f'} - ], - 'operationId': 'getJobResults', - 'responses': { - '200': {'$ref': '#/components/responses/200'}, - '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa - 'default': {'$ref': '#/components/responses/default'} # noqa - } - } - } - - tag = { - 'name': 'jobs', - 'description': 'Process jobs', - } - oas['tags'].insert(1, tag) - - oas['paths'] = paths - - if cfg['server'].get('admin', False): - schema_dict = get_config_schema() - oas['definitions'] = schema_dict['definitions'] - LOGGER.debug('Adding admin endpoints') - oas['paths'].update(get_admin()) - - return oas + return { + k: v + for k, v in collections.items() + if v.get('visibility', 'default') != 'hidden' + } -def get_config_schema() -> dict: - """ - Get configuration schema - - :returns: `dict` of configuration schema - """ - +def get_config_schema(): schema_file = os.path.join(THISDIR, 'schemas', 'config', 'pygeoapi-config-0.x.yml') @@ -777,12 +624,7 @@ def get_config_schema() -> dict: return yaml_load(fh2) -def get_admin() -> dict: - """ - Generate admin paths for OpenAPI Document - - :returns: `dict` of paths - """ +def get_admin(): schema_dict = get_config_schema() @@ -800,6 +642,7 @@ def get_admin() -> dict: ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict @@ -861,6 +704,7 @@ def get_admin() -> dict: ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa @@ -903,6 +747,7 @@ def get_admin() -> dict: ], 'responses': { '200': { + 'description': 'Successful response', 'content': { 'application/json': { 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa @@ -976,630 +821,6 @@ def get_admin() -> dict: return paths -def handle_collection(locale_: str, components: dict, collection_id: str, - collection_def: dict) -> Tuple[list, dict]: - """ - Generate relevant OpenAPI constructs for a given collection - - :param locale_: locale - :param components: OpenAPI components - :param collection_id: collection identifier - :param collection_def: collection definition - - :returns: `tuple` of `list` of tags and `dict` of paths - """ - - paths = {} - tags = [] - - items_f = deepcopy(components['parameters']['f']) - items_f['schema']['enum'].append('csv') - items_l = deepcopy(components['parameters']['lang']) - - if collection_def.get('visibility', 'default') == 'hidden': - LOGGER.debug(f'Skipping hidden layer: {collection_id}') - return [], {} - - name = l10n.translate(collection_id, locale_) - title = l10n.translate(collection_def['title'], locale_) - desc = l10n.translate(collection_def['description'], locale_) - collection_name_path = f'/collections/{collection_id}' - - tag = { - 'name': name, - 'description': desc, - 'externalDocs': {} - } - - for link in l10n.translate(collection_def.get('links', []), locale_): - if link['type'] == 'information': - tag['externalDocs']['description'] = link['type'] - tag['externalDocs']['url'] = link['url'] - break - - if len(tag['externalDocs']) == 0: - del tag['externalDocs'] - - tags.append(tag) - - paths[collection_name_path] = { - 'get': { - 'summary': f'Get {title} metadata', - 'description': desc, - 'tags': [name], - 'operationId': f'describe{name.capitalize()}Collection', - 'parameters': [ - {'$ref': '#/components/parameters/f'}, - {'$ref': '#/components/parameters/lang'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Collection"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - - LOGGER.debug('setting up collection endpoints') - try: - ptype = None - - if filter_providers_by_type(collection_def['providers'], 'feature'): - ptype = 'feature' - - if filter_providers_by_type(collection_def['providers'], 'record'): - ptype = 'record' - - p = load_plugin('provider', get_provider_by_type( - collection_def['providers'], ptype)) - - items_path = f'{collection_name_path}/items' - - coll_properties = deepcopy(components['parameters']['properties']) # noqa - - coll_properties['schema']['items']['enum'] = list(p.fields.keys()) - - paths[items_path] = { - 'get': { - 'summary': f'Get {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Features', - 'parameters': [ - items_f, - items_l, - {'$ref': '#/components/parameters/bbox'}, - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/limit"}, # noqa - {'$ref': '#/components/parameters/crs'}, # noqa - {'$ref': '#/components/parameters/bbox-crs'}, # noqa - coll_properties, - {'$ref': '#/components/parameters/vendorSpecificParameters'}, # noqa - {'$ref': '#/components/parameters/skipGeometry'}, - {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/sortby.yaml"}, # noqa - {'$ref': '#/components/parameters/offset'}, - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - }, - 'options': { - 'summary': f'Options for {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'options{name.capitalize()}Features', - 'responses': { - '200': {'description': 'options response'} - } - } - } - - if p.editable: - LOGGER.debug('Provider is editable; adding post') - - paths[items_path]['post'] = { - 'summary': f'Add {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'add{name.capitalize()}Features', - 'requestBody': { - 'description': 'Adds item to collection', - 'content': { - 'application/geo+json': { - 'schema': {} - } - }, - 'required': True - }, - 'responses': { - '201': {'description': 'Successful creation'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - - try: - schema_ref = p.get_schema(SchemaType.create) - paths[items_path]['post']['requestBody']['content'][schema_ref[0]] = { # noqa - 'schema': schema_ref[1] - } - except Exception as err: - LOGGER.debug(err) - - if ptype == 'record': - paths[items_path]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapir']}/parameters/q.yaml"}) - if p.fields: - schema_path = f'{collection_name_path}/schema' - - paths[schema_path] = { - 'get': { - 'summary': f'Get {title} schema', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Queryables', - 'parameters': [ - items_f, - items_l - ], - 'responses': { - '200': {'$ref': '#/components/responses/Queryables'}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa - } - } - } - - queryables_path = f'{collection_name_path}/queryables' - - paths[queryables_path] = { - 'get': { - 'summary': f'Get {title} queryables', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Queryables', - 'parameters': [ - items_f, - items_l - ], - 'responses': { - '200': {'$ref': '#/components/responses/Queryables'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa - } - } - } - - if p.time_field is not None: - paths[items_path]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa - - for field, type_ in p.fields.items(): - - if p.properties and field not in p.properties: - LOGGER.debug('Provider specified not to advertise property') - continue - - if field == 'q' and ptype == 'record': - LOGGER.debug('q parameter already declared, skipping') - continue - - if type_ == 'date': - schema = { - 'type': 'string', - 'format': 'date' - } - elif type_ == 'float': - schema = { - 'type': 'number', - 'format': 'float' - } - elif type_ == 'long': - schema = { - 'type': 'integer', - 'format': 'int64' - } - else: - schema = type_ - - path_ = f'{collection_name_path}/items' - paths[path_]['get']['parameters'].append({ - 'name': field, - 'in': 'query', - 'required': False, - 'schema': schema, - 'style': 'form', - 'explode': False - }) - - paths[f'{collection_name_path}/items/{{featureId}}'] = { - 'get': { - 'summary': f'Get {title} item by id', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Feature', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa - {'$ref': '#/components/parameters/crs'}, # noqa - {'$ref': '#/components/parameters/f'}, - {'$ref': '#/components/parameters/lang'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Feature"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - }, - 'options': { - 'summary': f'Options for {title} item by id', - 'description': desc, - 'tags': [name], - 'operationId': f'options{name.capitalize()}Feature', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa - ], - 'responses': { - '200': {'description': 'options response'} - } - } - } - - try: - schema_ref = p.get_schema() - paths[f'{collection_name_path}/items/{{featureId}}']['get']['responses']['200'] = { # noqa - 'content': { - schema_ref[0]: { - 'schema': schema_ref[1] - } - } - } - except Exception as err: - LOGGER.debug(err) - - if p.editable: - LOGGER.debug('Provider is editable; adding put/delete') - put_path = f'{collection_name_path}/items/{{featureId}}' - paths[put_path]['put'] = { # noqa - 'summary': f'Update {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'update{name.capitalize()}Features', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"} # noqa - ], - 'requestBody': { - 'description': 'Updates item in collection', - 'content': { - 'application/geo+json': { - 'schema': {} - } - }, - 'required': True - }, - 'responses': { - '204': {'$ref': '#/components/responses/204'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - - try: - schema_ref = p.get_schema(SchemaType.replace) - paths[put_path]['put']['requestBody']['content'][schema_ref[0]] = { # noqa - 'schema': schema_ref[1] - } - except Exception as err: - LOGGER.debug(err) - - paths[f'{collection_name_path}/items/{{featureId}}']['delete'] = { - 'summary': f'Delete {title} items', - 'description': desc, - 'tags': [name], - 'operationId': f'delete{name.capitalize()}Features', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/featureId"}, # noqa - ], - 'responses': { - '200': {'description': 'Successful delete'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - - except ProviderTypeError: - LOGGER.debug('collection is not feature based') - - LOGGER.debug('setting up coverage endpoints') - try: - load_plugin('provider', get_provider_by_type( - collection_def['providers'], 'coverage')) - - coverage_path = f'{collection_name_path}/coverage' - - paths[coverage_path] = { - 'get': { - 'summary': f'Get {title} coverage', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Coverage', - 'parameters': [ - items_f, - items_l, - {'$ref': '#/components/parameters/bbox'}, - {'$ref': '#/components/parameters/bbox-crs'}, - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - - except ProviderTypeError: - LOGGER.debug('collection is not coverage based') - - LOGGER.debug('setting up tiles endpoints') - tile_extension = filter_providers_by_type( - collection_def['providers'], 'tile') - - if tile_extension: - tp = load_plugin('provider', tile_extension) - - tiles_path = f'{collection_name_path}/tiles' - - paths[tiles_path] = { - 'get': { - 'summary': f'Fetch a {title} tiles description', - 'description': desc, - 'tags': [name], - 'operationId': f'describe{name.capitalize()}Tiles', - 'parameters': [ - items_f, - # items_l TODO: is this useful? - ], - 'responses': { - '200': {'$ref': '#/components/responses/Tiles'}, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - - tiles_data_path = f'{collection_name_path}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}' # noqa - - paths[tiles_data_path] = { - 'get': { - 'summary': f'Get a {title} tile', - 'description': desc, - 'tags': [name], - 'operationId': f'get{name.capitalize()}Tiles', - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrixSetId"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrix"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileRow"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileCol"}, # noqa - { - 'name': 'f', - 'in': 'query', - 'description': 'The optional f parameter indicates the output format which the server shall provide as part of the response document.', # noqa - 'required': False, - 'schema': { - 'type': 'string', - 'enum': [tp.format_type], - 'default': tp.format_type - }, - 'style': 'form', - 'explode': False - } - ], - 'responses': { - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '404': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/NotFound"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - mimetype = tile_extension['format']['mimetype'] - paths[tiles_data_path]['get']['responses']['200'] = { - 'description': 'successful operation', - 'content': { - mimetype: { - 'schema': { - 'type': 'string', - 'format': 'binary' - } - } - } - } - - LOGGER.debug('setting up edr endpoints') - edr_extension = filter_providers_by_type( - collection_def['providers'], 'edr') - - if edr_extension: - ep = load_plugin('provider', edr_extension) - - edr_query_endpoints = [] - - for qt in [qt for qt in ep.get_query_types() if qt != 'locations']: - edr_query_endpoints.append({ - 'path': f'{collection_name_path}/{qt}', - 'qt': qt, - 'op_id': f'query{qt.capitalize()}{collection_id.capitalize()}' - }) - if ep.instances: - edr_query_endpoints.append({ - 'path': f'{collection_name_path}/instances/{{instanceId}}/{qt}', # noqa - 'qt': qt, - 'op_id': f'query{qt.capitalize()}Instance{collection_id.capitalize()}' # noqa - }) - - for eqe in edr_query_endpoints: - if eqe['qt'] == 'cube': - spatial_parameter = 'bbox' - else: - spatial_parameter = f"{eqe['qt']}Coords" - paths[eqe['path']] = { - 'get': { - 'summary': f"query {collection_def['description']} by {eqe['qt']}", # noqa - 'description': collection_def['description'], - 'tags': [collection_id], - 'operationId': eqe['op_id'], - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': { - 'description': 'Response', - 'content': { - 'application/prs.coverage+json': { - 'schema': { - '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa - } - } - } - } - } - } - } - if 'locations' in ep.get_query_types(): - paths[f'{collection_name_path}/locations'] = { - 'get': { - 'summary': f"Get pre-defined locations of {v['description']}", # noqa - 'description': collection_def['description'], - 'tags': [collection_id], - 'operationId': f'queryLOCATIONS{collection_id.capitalize()}', # noqa - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/bbox.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/Features"}, # noqa - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa - } - } - } - paths[f'{collection_name_path}/locations/{{locId}}'] = { - 'get': { - 'summary': f"query {collection_defv['description']} by location", # noqa - 'description': collection_def['description'], - 'tags': [collection_id], - 'operationId': f'queryLOCATIONSBYID{collection_id.capitalize()}', # noqa - 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/{spatial_parameter}.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/locationId.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/parameter-name.yaml"}, # noqa - {'$ref': f"{OPENAPI_YAML['oaedr']}/parameters/z.yaml"}, - {'$ref': '#/components/parameters/f'} - ], - 'responses': { - '200': { - 'description': 'Response', - 'content': { - 'application/prs.coverage+json': { - 'schema': { - '$ref': f"{OPENAPI_YAML['oaedr']}/schemas/coverageJSON.yaml" # noqa - } - } - } - } - } - } - } - - LOGGER.debug('setting up maps endpoints') - map_extension = filter_providers_by_type( - collection_def['providers'], 'map') - - if map_extension: - mp = load_plugin('provider', map_extension) - - map_f = deepcopy(components['parameters']['f']) - map_f['schema']['enum'] = [map_extension['format']['name']] - map_f['schema']['default'] = map_extension['format']['name'] - - pth = f'/collections/{collection_def}/map' - paths[pth] = { - 'get': { - 'summary': 'Get map', - 'description': f"{collection_def['description']} map", - 'tags': [collection_id], - 'operationId': 'getMap', - 'parameters': [ - {'$ref': '#/components/parameters/bbox'}, - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}, # noqa - { - 'name': 'width', - 'in': 'query', - 'description': 'Response image width', - 'required': False, - 'schema': { - 'type': 'integer', - }, - 'style': 'form', - 'explode': False - }, - { - 'name': 'height', - 'in': 'query', - 'description': 'Response image height', - 'required': False, - 'schema': { - 'type': 'integer', - }, - 'style': 'form', - 'explode': False - }, - { - 'name': 'transparent', - 'in': 'query', - 'description': 'Background transparency of map (default=true).', # noqa - 'required': False, - 'schema': { - 'type': 'boolean', - 'default': True, - }, - 'style': 'form', - 'explode': False - }, - {'$ref': '#/components/parameters/bbox-crs-epsg'}, - map_f - ], - 'responses': { - '200': { - 'description': 'Response', - 'content': { - 'application/json': {} - } - }, - '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa - '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"}, # noqa - } - } - } - if mp.time_field is not None: - paths[pth]['get']['parameters'].append( - {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/parameters/datetime"}) # noqa - - return tags, paths - - def get_oas(cfg: dict, fail_on_invalid_collection: bool = True, version='3.0') -> dict: """ @@ -1647,9 +868,9 @@ def generate_openapi_document(cfg_file: Union[Path, io.TextIOWrapper], :param cfg_file: configuration Path instance (`str` of filepath or parsed `dict`) + :param output_format: output format for OpenAPI document :param fail_on_invalid_collection: `bool` of whether to fail on an invalid collection - :param output_format: output format for OpenAPI document :returns: `str` of the OpenAPI document in the output format requested """ @@ -1670,7 +891,6 @@ def generate_openapi_document(cfg_file: Union[Path, io.TextIOWrapper], content = yaml.safe_dump(oas, default_flow_style=False) else: content = to_json(oas, pretty=pretty_print) - return content diff --git a/pygeoapi/process/manager/base.py b/pygeoapi/process/manager/base.py index ad208f3..d4d8f23 100644 --- a/pygeoapi/process/manager/base.py +++ b/pygeoapi/process/manager/base.py @@ -52,7 +52,7 @@ from pygeoapi.util import ( JobStatus, ProcessExecutionMode, RequestedProcessExecutionMode, - Subscriber, + Subscriber ) LOGGER = logging.getLogger(__name__) @@ -318,7 +318,6 @@ class BaseManager: jfmt = 'application/json' self.update_job(job_id, job_metadata) - self._send_failed_notification(subscriber) return jfmt, outputs, current_status @@ -328,7 +327,7 @@ class BaseManager: process_id: str, data_dict: dict, execution_mode: Optional[RequestedProcessExecutionMode] = None, - subscriber: Optional[Subscriber] = None, + subscriber: Optional[Subscriber] = None ) -> Tuple[str, Any, JobStatus, Optional[Dict[str, str]]]: """ Default process execution handler @@ -390,6 +389,7 @@ class BaseManager: # managers **({'subscriber': subscriber} if self.supports_subscribing else {}) ) + return job_id, mime_type, outputs, status, response_headers def _send_in_progress_notification(self, subscriber: Optional[Subscriber]): diff --git a/pygeoapi/process/manager/dummy.py b/pygeoapi/process/manager/dummy.py index 6676617..eb35eee 100644 --- a/pygeoapi/process/manager/dummy.py +++ b/pygeoapi/process/manager/dummy.py @@ -72,7 +72,7 @@ class DummyManager(BaseManager): process_id: str, data_dict: dict, execution_mode: Optional[RequestedProcessExecutionMode] = None, - subscriber: Optional[Subscriber] = None, + subscriber: Optional[Subscriber] = None ) -> Tuple[str, str, Any, JobStatus, Optional[Dict[str, str]]]: """ Default process execution handler diff --git a/pygeoapi/provider/xarray_.py b/pygeoapi/provider/xarray_.py index d1595c4..2609e29 100644 --- a/pygeoapi/provider/xarray_.py +++ b/pygeoapi/provider/xarray_.py @@ -398,12 +398,15 @@ class XarrayProvider(BaseProvider): } if 'crs' in self._data.variables.keys(): - properties['bbox_crs'] = f'http://www.opengis.net/def/crs/OGC/1.3/{self._data.crs.epsg_code}' # noqa + try: + properties['bbox_crs'] = f'http://www.opengis.net/def/crs/OGC/1.3/{self._data.crs.epsg_code}' # noqa - properties['inverse_flattening'] = self._data.crs.\ - inverse_flattening + properties['inverse_flattening'] = self._data.crs.\ + inverse_flattening - properties['crs_type'] = 'ProjectedCRS' + properties['crs_type'] = 'ProjectedCRS' + except AttributeError: + pass properties['axes'] = [ properties['x_axis_label'], diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 8264566..6d6b705 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -50,7 +50,14 @@ from starlette.responses import ( ) import uvicorn -from pygeoapi.api import API +from pygeoapi.api import API, APIRequest, apply_gzip +import pygeoapi.api.coverages as coverages_api +import pygeoapi.api.environmental_data_retrieval as edr_api +import pygeoapi.api.itemtypes as itemtypes_api +import pygeoapi.api.maps as maps_api +import pygeoapi.api.processes as processes_api +import pygeoapi.api.stac as stac_api +import pygeoapi.api.tiles as tiles_api from pygeoapi.openapi import load_openapi_document from pygeoapi.config import get_config from pygeoapi.util import get_api_rules @@ -113,10 +120,12 @@ async def get_response( """ loop = asyncio.get_running_loop() - result = await loop.run_in_executor( + headers, status, content = await loop.run_in_executor( None, call_api_threadsafe, loop, api_call, *args) + return _to_response(headers, status, content) - headers, status, content = result + +def _to_response(headers, status, content): if headers['Content-Type'] == 'text/html': response = HTMLResponse(content=content, status_code=status) else: @@ -130,6 +139,27 @@ async def get_response( return response +async def execute_from_starlette(api_function, request: Request, *args, + skip_valid_check=False) -> Response: + api_request = await APIRequest.from_starlette(request, api_.locales) + content: str | bytes + if not skip_valid_check and not api_request.is_valid(): + headers, status, content = api_.get_format_exception(api_request) + else: + + loop = asyncio.get_running_loop() + headers, status, content = await loop.run_in_executor( + None, call_api_threadsafe, loop, api_function, + api_, api_request, *args) + # NOTE: that gzip currently doesn't work in starlette + # https://github.com/geopython/pygeoapi/issues/1591 + content = apply_gzip(headers, content) + + response = _to_response(headers, status, content) + + return response + + async def landing_page(request: Request): """ OGC API landing page endpoint @@ -173,8 +203,9 @@ async def get_tilematrix_set(request: Request, tileMatrixSetId=None): if 'tileMatrixSetId' in request.path_params: tileMatrixSetId = request.path_params['tileMatrixSetId'] - return await get_response( - api_.tilematrixset, request, tileMatrixSetId) + return await execute_from_starlette( + tiles_api.tilematrixset, request, tileMatrixSetId, + ) async def get_tilematrix_sets(request: Request): @@ -183,7 +214,7 @@ async def get_tilematrix_sets(request: Request): :returns: HTTP response """ - return await get_response(api_.tilematrixsets, request) + return await execute_from_starlette(tiles_api.tilematrixsets, request) async def collection_schema(request: Request, collection_id=None): @@ -198,8 +229,8 @@ async def collection_schema(request: Request, collection_id=None): if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_schema, request, collection_id) + return await get_response(api_.get_collection_schema, request, + collection_id) async def collection_queryables(request: Request, collection_id=None): @@ -214,8 +245,9 @@ async def collection_queryables(request: Request, collection_id=None): if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_queryables, request, collection_id) + return await execute_from_starlette( + itemtypes_api.get_collection_queryables, request, collection_id, + ) async def get_collection_tiles(request: Request, collection_id=None): @@ -229,8 +261,9 @@ async def get_collection_tiles(request: Request, collection_id=None): """ if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_tiles, request, collection_id) + + return await execute_from_starlette( + tiles_api.get_collection_tiles, request, collection_id) async def get_collection_tiles_metadata(request: Request, collection_id=None, @@ -247,9 +280,10 @@ async def get_collection_tiles_metadata(request: Request, collection_id=None, collection_id = request.path_params['collection_id'] if 'tileMatrixSetId' in request.path_params: tileMatrixSetId = request.path_params['tileMatrixSetId'] - return await get_response( - api_.get_collection_tiles_metadata, request, - collection_id, tileMatrixSetId + + return await execute_from_starlette( + tiles_api.get_collection_tiles_metadata, request, + collection_id, tileMatrixSetId, skip_valid_check=True, ) @@ -278,9 +312,10 @@ async def get_collection_items_tiles(request: Request, collection_id=None, tileRow = request.path_params['tileRow'] if 'tileCol' in request.path_params: tileCol = request.path_params['tileCol'] - return await get_response( - api_.get_collection_tiles_data, request, collection_id, - tileMatrixSetId, tile_matrix, tileRow, tileCol + return await execute_from_starlette( + tiles_api.get_collection_tiles_data, request, collection_id, + tileMatrixSetId, tile_matrix, tileRow, tileCol, + skip_valid_check=True, ) @@ -301,45 +336,47 @@ async def collection_items(request: Request, collection_id=None, item_id=None): item_id = request.path_params['item_id'] if item_id is None: if request.method == 'GET': # list items - return await get_response( - api_.get_collection_items, request, collection_id) + return await execute_from_starlette( + itemtypes_api.get_collection_items, request, collection_id, + skip_valid_check=True) elif request.method == 'POST': # filter or manage items content_type = request.headers.get('content-type') if content_type is not None: if content_type == 'application/geo+json': - return await get_response( - api_.manage_collection_item, request, - 'create', collection_id) + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, + 'create', collection_id, skip_valid_check=True) else: - return await get_response( - api_.post_collection_items, + return await execute_from_starlette( + itemtypes_api.post_collection_items, request, - collection_id + collection_id, + skip_valid_check=True, ) elif request.method == 'OPTIONS': - return await get_response( - api_.manage_collection_item, request, - 'options', collection_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, + 'options', collection_id, skip_valid_check=True, ) elif request.method == 'DELETE': - return await get_response( - api_.manage_collection_item, request, 'delete', - collection_id, item_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'delete', + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'PUT': - return await get_response( - api_.manage_collection_item, request, 'update', - collection_id, item_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'update', + collection_id, item_id, skip_valid_check=True, ) elif request.method == 'OPTIONS': - return await get_response( - api_.manage_collection_item, request, 'options', - collection_id, item_id + return await execute_from_starlette( + itemtypes_api.manage_collection_item, request, 'options', + collection_id, item_id, skip_valid_check=True, ) else: - return await get_response( - api_.get_collection_item, request, collection_id, item_id) + return await execute_from_starlette( + itemtypes_api.get_collection_item, request, collection_id, item_id) async def collection_coverage(request: Request, collection_id=None): @@ -354,8 +391,8 @@ async def collection_coverage(request: Request, collection_id=None): if 'collection_id' in request.path_params: collection_id = request.path_params['collection_id'] - return await get_response( - api_.get_collection_coverage, request, collection_id) + return await execute_from_starlette( + coverages_api.get_collection_coverage, request, collection_id) async def collection_map(request: Request, collection_id, style_id=None): @@ -373,8 +410,9 @@ async def collection_map(request: Request, collection_id, style_id=None): if 'style_id' in request.path_params: style_id = request.path_params['style_id'] - return await get_response( - api_.get_collection_map, request, collection_id, style_id) + return await execute_from_starlette( + maps_api.get_collection_map, request, collection_id, style_id + ) async def get_processes(request: Request, process_id=None): @@ -389,7 +427,8 @@ async def get_processes(request: Request, process_id=None): if 'process_id' in request.path_params: process_id = request.path_params['process_id'] - return await get_response(api_.describe_processes, request, process_id) + return await execute_from_starlette(processes_api.describe_processes, + request, process_id) async def get_jobs(request: Request, job_id=None): @@ -406,12 +445,14 @@ async def get_jobs(request: Request, job_id=None): job_id = request.path_params['job_id'] if job_id is None: # list of submit job - return await get_response(api_.get_jobs, request) + return await execute_from_starlette(processes_api.get_jobs, request) else: # get or delete job if request.method == 'DELETE': - return await get_response(api_.delete_job, job_id) + return await execute_from_starlette(processes_api.delete_job, + request, job_id) else: # Return status of a specific job - return await get_response(api_.get_jobs, request, job_id) + return await execute_from_starlette(processes_api.get_jobs, + request, job_id) async def execute_process_jobs(request: Request, process_id=None): @@ -427,7 +468,8 @@ async def execute_process_jobs(request: Request, process_id=None): if 'process_id' in request.path_params: process_id = request.path_params['process_id'] - return await get_response(api_.execute_process, request, process_id) + return await execute_from_starlette(processes_api.execute_process, + request, process_id) async def get_job_result(request: Request, job_id=None): @@ -443,7 +485,8 @@ async def get_job_result(request: Request, job_id=None): if 'job_id' in request.path_params: job_id = request.path_params['job_id'] - return await get_response(api_.get_job_result, request, job_id) + return await execute_from_starlette(processes_api.get_job_result, + request, job_id) async def get_job_result_resource(request: Request, @@ -463,6 +506,7 @@ async def get_job_result_resource(request: Request, if 'resource' in request.path_params: resource = request.path_params['resource'] + # TODO: this api function currently doesn't exist return await get_response( api_.get_job_result_resource, request, job_id, resource) @@ -484,9 +528,10 @@ async def get_collection_edr_query(request: Request, collection_id=None, instanc instance_id = request.path_params['instance_id'] query_type = request["path"].split('/')[-1] # noqa - return await get_response( - api_.get_collection_edr_query, request, collection_id, - instance_id, query_type + return await execute_from_starlette( + edr_api.get_collection_edr_query, request, collection_id, + instance_id, query_type, + skip_valid_check=True, ) @@ -513,7 +558,7 @@ async def stac_catalog_root(request: Request): :returns: Starlette HTTP response """ - return await get_response(api_.get_stac_root, request) + return await execute_from_starlette(stac_api.get_stac_root, request) async def stac_catalog_path(request: Request): @@ -525,7 +570,7 @@ async def stac_catalog_path(request: Request): :returns: Starlette HTTP response """ path = request.path_params["path"] - return await get_response(api_.get_stac_path, request, path) + return await execute_from_starlette(stac_api.get_stac_path, request, path) async def admin_config(request: Request): diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 2ccbeb2..7e1f64e 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -609,10 +609,12 @@ class JobStatus(Enum): @dataclass(frozen=True) class Subscriber: - """Store subscriber urls as defined in: + """ + Store subscriber URLs as defined in: https://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi/schemas/subscriber.yaml # noqa """ + success_uri: str in_progress_uri: Optional[str] failed_uri: Optional[str] diff --git a/tests/api/test_api.py b/tests/api/test_api.py new file mode 100644 index 0000000..c11afca --- /dev/null +++ b/tests/api/test_api.py @@ -0,0 +1,869 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import json +import gzip +from http import HTTPStatus + +from pyld import jsonld +import pytest + +from pygeoapi.api import ( + API, APIRequest, FORMAT_TYPES, F_HTML, F_JSON, F_JSONLD, F_GZIP, + __version__, validate_bbox, validate_datetime, + validate_subset +) +from pygeoapi.util import yaml_load, get_api_rules, get_base_url + +from tests.util import (get_test_file_path, mock_flask, mock_starlette, + mock_request) + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_with_rules() -> dict: + """ Returns a pygeoapi configuration with default API rules. """ + with open(get_test_file_path('pygeoapi-test-config-apirules.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_enclosure() -> dict: + """ Returns a pygeoapi configuration with enclosure links. """ + with open(get_test_file_path('pygeoapi-test-config-enclosure.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def config_hidden_resources(): + filename = 'pygeoapi-test-config-hidden-resources.yml' + with open(get_test_file_path(filename)) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def enclosure_api(config_enclosure, openapi): + """ Returns an API instance with a collection with enclosure links. """ + return API(config_enclosure, openapi) + + +@pytest.fixture() +def rules_api(config_with_rules, openapi): + """ Returns an API instance with URL prefix and strict slashes policy. + The API version is extracted from the current version here. + """ + return API(config_with_rules, openapi) + + +@pytest.fixture() +def api_hidden_resources(config_hidden_resources, openapi): + return API(config_hidden_resources, openapi) + + +def test_apirequest(api_): + # Test without (valid) locales + with pytest.raises(ValueError): + req = mock_request() + APIRequest(req, []) + APIRequest(req, None) + APIRequest(req, ['zz']) + + # Test all supported formats from query args + for f, mt in FORMAT_TYPES.items(): + req = mock_request({'f': f}) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == f + assert apireq.get_response_headers()['Content-Type'] == mt + + # Test all supported formats from Accept header + for f, mt in FORMAT_TYPES.items(): + req = mock_request(HTTP_ACCEPT=mt) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == f + assert apireq.get_response_headers()['Content-Type'] == mt + + # Test nonsense format + req = mock_request({'f': 'foo'}) + apireq = APIRequest(req, api_.locales) + assert not apireq.is_valid() + assert apireq.format == 'foo' + assert apireq.is_valid(('foo',)) + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSON] + + # Test without format + req = mock_request() + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format is None + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSON] + assert apireq.get_linkrel(F_JSON) == 'self' + assert apireq.get_linkrel(F_HTML) == 'alternate' + + # Test complex format string + hh = 'text/html,application/xhtml+xml,application/xml;q=0.9,' + req = mock_request(HTTP_ACCEPT=hh) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_HTML + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_HTML] + assert apireq.get_linkrel(F_HTML) == 'self' + assert apireq.get_linkrel(F_JSON) == 'alternate' + + # Test accept header with multiple valid formats + hh = 'plain/text,application/ld+json,application/json;q=0.9,' + req = mock_request(HTTP_ACCEPT=hh) + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_JSONLD + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_JSONLD] + assert apireq.get_linkrel(F_JSONLD) == 'self' + assert apireq.get_linkrel(F_HTML) == 'alternate' + + # Overrule HTTP content negotiation + req = mock_request({'f': 'html'}, HTTP_ACCEPT='application/json') # noqa + apireq = APIRequest(req, api_.locales) + assert apireq.is_valid() + assert apireq.format == F_HTML + assert apireq.get_response_headers()['Content-Type'] == \ + FORMAT_TYPES[F_HTML] + + # Test data + for d in (None, '', 'test', {'key': 'value'}): + req = mock_request(data=d) + apireq = APIRequest.with_data(req, api_.locales) + if not d: + assert apireq.data == b'' + elif isinstance(d, dict): + assert d == json.loads(apireq.data) + else: + assert apireq.data == d.encode() + + # Test multilingual + test_lang = { + 'nl': ('en', 'en-US'), # unsupported lang should return default + 'en-US': ('en', 'en-US'), + 'de_CH': ('en', 'en-US'), + 'fr-CH, fr;q=0.9, en;q=0.8': ('fr', 'fr-CA'), + 'fr-CH, fr-BE;q=0.9': ('fr', 'fr-CA'), + } + sup_lang = ('en-US', 'fr_CA') + for lang_in, (lang_out, cl_out) in test_lang.items(): + # Using l query parameter + req = mock_request({'lang': lang_in}) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == lang_in + assert apireq.locale.language == lang_out + assert apireq.get_response_headers()['Content-Language'] == cl_out + + # Using Accept-Language header + req = mock_request(HTTP_ACCEPT_LANGUAGE=lang_in) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == lang_in + assert apireq.locale.language == lang_out + assert apireq.get_response_headers()['Content-Language'] == cl_out + + # Test language override + req = mock_request({'lang': 'fr'}, HTTP_ACCEPT_LANGUAGE='en_US') + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == 'fr' + assert apireq.locale.language == 'fr' + assert apireq.get_response_headers()['Content-Language'] == 'fr-CA' + + # Test locale territory + req = mock_request({'lang': 'en-GB'}) + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale == 'en-GB' + assert apireq.locale.language == 'en' + assert apireq.locale.territory == 'US' + assert apireq.get_response_headers()['Content-Language'] == 'en-US' + + # Test without Accept-Language header or 'lang' query parameter + # (should return default language from YAML config) + req = mock_request() + apireq = APIRequest(req, api_.locales) + assert apireq.raw_locale is None + assert apireq.locale.language == api_.default_locale.language + assert apireq.get_response_headers()['Content-Language'] == 'en-US' + + # Test without Accept-Language header or 'lang' query param + # (should return first in custom list of languages) + sup_lang = ('de', 'fr', 'en') + apireq = APIRequest(req, sup_lang) + assert apireq.raw_locale is None + assert apireq.locale.language == 'de' + assert apireq.get_response_headers()['Content-Language'] == 'de' + + +def test_apirules_active(config_with_rules, rules_api): + assert rules_api.config == config_with_rules + rules = get_api_rules(config_with_rules) + base_url = get_base_url(config_with_rules) + + # Test Flask + flask_prefix = rules.get_url_prefix('flask') + with mock_flask('pygeoapi-test-config-apirules.yml') as flask_client: + # Test happy path + response = flask_client.get(f'{flask_prefix}/conformance') + assert response.status_code == 200 + assert response.headers['X-API-Version'] == __version__ + assert response.request.url == \ + flask_client.application.url_for('pygeoapi.conformance') + response = flask_client.get(f'{flask_prefix}/static/img/pygeoapi.png') + assert response.status_code == 200 + # Test that static resources also work without URL prefix + response = flask_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test strict slashes + response = flask_client.get(f'{flask_prefix}/conformance/') + assert response.status_code == 404 + # For the landing page ONLY, trailing slashes are actually preferred. + # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa + # Omitting the trailing slash should lead to a redirect. + response = flask_client.get(f'{flask_prefix}/') + assert response.status_code == 200 + response = flask_client.get(flask_prefix) + assert response.status_code in (307, 308) + + # Test links on landing page for correct URLs + response = flask_client.get(flask_prefix, follow_redirects=True) + assert response.status_code == 200 + assert response.is_json + links = response.json['links'] + assert all( + href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + ) + + # Test Starlette + starlette_prefix = rules.get_url_prefix('starlette') + with mock_starlette('pygeoapi-test-config-apirules.yml') as starlette_client: # noqa + # Test happy path + response = starlette_client.get(f'{starlette_prefix}/conformance') + assert response.status_code == 200 + assert response.headers['X-API-Version'] == __version__ + response = starlette_client.get(f'{starlette_prefix}/static/img/pygeoapi.png') # noqa + assert response.status_code == 200 + # Test that static resources also work without URL prefix + response = starlette_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test strict slashes + response = starlette_client.get(f'{starlette_prefix}/conformance/') + assert response.status_code == 404 + # For the landing page ONLY, trailing slashes are actually preferred. + # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa + # Omitting the trailing slash should lead to a redirect. + response = starlette_client.get(f'{starlette_prefix}/') + assert response.status_code == 200 + response = starlette_client.get(starlette_prefix) + assert response.status_code in (307, 308) + + # Test links on landing page for correct URLs + response = starlette_client.get(starlette_prefix, follow_redirects=True) # noqa + assert response.status_code == 200 + links = response.json()['links'] + assert all( + href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa + ) + + +def test_apirules_inactive(config, api_): + assert api_.config == config + rules = get_api_rules(config) + + # Test Flask + flask_prefix = rules.get_url_prefix('flask') + assert flask_prefix == '' + with mock_flask('pygeoapi-test-config.yml') as flask_client: + response = flask_client.get('') + assert response.status_code == 200 + response = flask_client.get('/conformance') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + assert response.request.url == \ + flask_client.application.url_for('pygeoapi.conformance') + response = flask_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test trailing slashes + response = flask_client.get('/') + assert response.status_code == 200 + response = flask_client.get('/conformance/') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + + # Test Starlette + starlette_prefix = rules.get_url_prefix('starlette') + assert starlette_prefix == '' + with mock_starlette('pygeoapi-test-config.yml') as starlette_client: + response = starlette_client.get('') + assert response.status_code == 200 + response = starlette_client.get('/conformance') + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + assert str(response.url) == f"{starlette_client.base_url}/conformance" + response = starlette_client.get('/static/img/pygeoapi.png') + assert response.status_code == 200 + + # Test trailing slashes + response = starlette_client.get('/') + assert response.status_code == 200 + response = starlette_client.get('/conformance/', follow_redirects=True) + assert response.status_code == 200 + assert 'X-API-Version' not in response.headers + + +def test_api(config, api_, openapi): + assert api_.config == config + assert isinstance(api_.config, dict) + + req = mock_request(HTTP_ACCEPT='application/json') + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == 'application/vnd.oai.openapi+json;version=3.0' # noqa + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + root = json.loads(response) + assert isinstance(root, dict) + + a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + req = mock_request(HTTP_ACCEPT=a) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ + FORMAT_TYPES[F_HTML] + + assert 'Swagger UI' in response + + a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' + req = mock_request({'ui': 'redoc'}, HTTP_ACCEPT=a) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ + FORMAT_TYPES[F_HTML] + + assert 'ReDoc' in response + + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.openapi_(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + assert api_.get_collections_url() == 'http://localhost:5000/collections' + + +def test_api_exception(config, api_): + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + # When a language is set, the exception should still be English + req = mock_request({'f': 'foo', 'lang': 'fr'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Language'] == 'en-US' + assert code == HTTPStatus.BAD_REQUEST + + +def test_gzip(config, api_, openapi): + # Requests for each response type and gzip encoding + req_gzip_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML], + HTTP_ACCEPT_ENCODING=F_GZIP) + req_gzip_gzip = mock_request(HTTP_ACCEPT='application/gzip', + HTTP_ACCEPT_ENCODING=F_GZIP) + + # Responses from server config without gzip compression + rsp_headers, _, rsp_json = api_.landing_page(req_gzip_json) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + rsp_headers, _, rsp_jsonld = api_.landing_page(req_gzip_jsonld) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + rsp_headers, _, rsp_html = api_.landing_page(req_gzip_html) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + rsp_headers, _, _ = api_.landing_page(req_gzip_gzip) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + + # Add gzip to server and use utf-16 encoding + config['server']['gzip'] = True + enc_16 = 'utf-16' + config['server']['encoding'] = enc_16 + api_ = API(config, openapi) + + # Responses from server with gzip compression + rsp_json_headers, _, rsp_gzip_json = api_.landing_page(req_gzip_json) + rsp_jsonld_headers, _, rsp_gzip_jsonld = api_.landing_page(req_gzip_jsonld) + rsp_html_headers, _, rsp_gzip_html = api_.landing_page(req_gzip_html) + rsp_gzip_headers, _, rsp_gzip_gzip = api_.landing_page(req_gzip_gzip) + + # Validate compressed json response + assert rsp_json_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_JSON]}; charset={enc_16}' + assert rsp_json_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_json = gzip.decompress(rsp_gzip_json).decode(enc_16) + assert isinstance(parsed_gzip_json, str) + parsed_gzip_json = json.loads(parsed_gzip_json) + assert isinstance(parsed_gzip_json, dict) + assert parsed_gzip_json == json.loads(rsp_json) + + # Validate compressed jsonld response + assert rsp_jsonld_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_JSONLD]}; charset={enc_16}' + assert rsp_jsonld_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_jsonld = gzip.decompress(rsp_gzip_jsonld).decode(enc_16) + assert isinstance(parsed_gzip_jsonld, str) + parsed_gzip_jsonld = json.loads(parsed_gzip_jsonld) + assert isinstance(parsed_gzip_jsonld, dict) + assert parsed_gzip_jsonld == json.loads(rsp_jsonld) + + # Validate compressed html response + assert rsp_html_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_HTML]}; charset={enc_16}' + assert rsp_html_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_html = gzip.decompress(rsp_gzip_html).decode(enc_16) + assert isinstance(parsed_gzip_html, str) + assert parsed_gzip_html == rsp_html + + # Validate compressed gzip response + assert rsp_gzip_headers['Content-Type'] == \ + f'{FORMAT_TYPES[F_GZIP]}; charset={enc_16}' + assert rsp_gzip_headers['Content-Encoding'] == F_GZIP + + parsed_gzip_gzip = gzip.decompress(rsp_gzip_gzip).decode(enc_16) + assert isinstance(parsed_gzip_gzip, str) + parsed_gzip_gzip = json.loads(parsed_gzip_gzip) + assert isinstance(parsed_gzip_gzip, dict) + + # Requests without content encoding header + req_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON]) + req_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD]) + req_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML]) + + # Responses without content encoding + _, _, rsp_json_ = api_.landing_page(req_json) + _, _, rsp_jsonld_ = api_.landing_page(req_jsonld) + _, _, rsp_html_ = api_.landing_page(req_html) + + # Confirm each request is the same when decompressed + assert rsp_json_ == rsp_json == \ + gzip.decompress(rsp_gzip_json).decode(enc_16) + + assert rsp_jsonld_ == rsp_jsonld == \ + gzip.decompress(rsp_gzip_jsonld).decode(enc_16) + + assert rsp_html_ == rsp_html == \ + gzip.decompress(rsp_gzip_html).decode(enc_16) + + +def test_root(config, api_): + req = mock_request() + rsp_headers, code, response = api_.landing_page(req) + root = json.loads(response) + + assert rsp_headers['Content-Type'] == 'application/json' == \ + FORMAT_TYPES[F_JSON] + assert rsp_headers['X-Powered-By'].startswith('pygeoapi') + assert rsp_headers['Content-Language'] == 'en-US' + + assert isinstance(root, dict) + assert 'links' in root + assert root['links'][0]['rel'] == 'self' + assert root['links'][0]['type'] == FORMAT_TYPES[F_JSON] + assert root['links'][0]['href'].endswith('?f=json') + assert any(link['href'].endswith('f=jsonld') and link['rel'] == 'alternate' + for link in root['links']) + assert any(link['href'].endswith('f=html') and link['rel'] == 'alternate' + for link in root['links']) + assert len(root['links']) == 11 + assert 'title' in root + assert root['title'] == 'pygeoapi default instance' + assert 'description' in root + assert root['description'] == 'pygeoapi provides an API to geospatial data' + + req = mock_request({'f': 'html'}) + rsp_headers, code, response = api_.landing_page(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_root_structured_data(config, api_): + req = mock_request({"f": "jsonld"}) + rsp_headers, code, response = api_.landing_page(req) + root = json.loads(response) + + assert rsp_headers['Content-Type'] == 'application/ld+json' == \ + FORMAT_TYPES[F_JSONLD] + assert rsp_headers['Content-Language'] == 'en-US' + assert rsp_headers['X-Powered-By'].startswith('pygeoapi') + + assert isinstance(root, dict) + assert 'description' in root + assert root['description'] == 'pygeoapi provides an API to geospatial data' + + assert '@context' in root + assert root['@context'] == 'https://schema.org/docs/jsonldcontext.jsonld' + expanded = jsonld.expand(root)[0] + assert '@type' in expanded + assert 'http://schema.org/DataCatalog' in expanded['@type'] + assert 'http://schema.org/description' in expanded + assert root['description'] == expanded['http://schema.org/description'][0][ + '@value'] + assert 'http://schema.org/keywords' in expanded + assert len(expanded['http://schema.org/keywords']) == 3 + assert '@value' in expanded['http://schema.org/keywords'][0].keys() + assert 'http://schema.org/provider' in expanded + assert expanded['http://schema.org/provider'][0]['@type'][ + 0] == 'http://schema.org/Organization' + assert expanded['http://schema.org/name'][0]['@value'] == root['name'] + + +def test_conformance(config, api_): + req = mock_request() + rsp_headers, code, response = api_.conformance(req) + root = json.loads(response) + + assert isinstance(root, dict) + assert 'conformsTo' in root + assert len(root['conformsTo']) == 37 + assert 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs' \ + in root['conformsTo'] + + req = mock_request({'f': 'foo'}) + rsp_headers, code, response = api_.conformance(req) + assert code == HTTPStatus.BAD_REQUEST + + req = mock_request({'f': 'html'}) + rsp_headers, code, response = api_.conformance(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_describe_collections(config, api_): + req = mock_request({"f": "foo"}) + rsp_headers, code, response = api_.describe_collections(req) + assert code == HTTPStatus.BAD_REQUEST + + req = mock_request({"f": "html"}) + rsp_headers, code, response = api_.describe_collections(req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_request() + rsp_headers, code, response = api_.describe_collections(req) + collections = json.loads(response) + + assert len(collections) == 2 + assert len(collections['collections']) == 9 + assert len(collections['links']) == 3 + + rsp_headers, code, response = api_.describe_collections(req, 'foo') + collection = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert rsp_headers['Content-Language'] == 'en-US' + assert collection['id'] == 'obs' + assert collection['title'] == 'Observations' + assert collection['description'] == 'My cool observations' + assert len(collection['links']) == 14 + assert collection['extent'] == { + 'spatial': { + 'bbox': [[-180, -90, 180, 90]], + 'crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + }, + 'temporal': { + 'interval': [ + ['2000-10-30T18:24:39+00:00', '2007-10-30T08:57:29+00:00'] + ], + 'trs': 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' + } + } + + # OAPIF Part 2 CRS 6.2.1 A, B, configured CRS + defaults + assert collection['crs'] is not None + crs_set = [ + 'http://www.opengis.net/def/crs/EPSG/0/28992', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/EPSG/0/4326', + ] + for crs in crs_set: + assert crs in collection['crs'] + assert collection['storageCRS'] is not None + assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa + assert 'storageCrsCoordinateEpoch' not in collection + + # French language request + req = mock_request({'lang': 'fr'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert rsp_headers['Content-Language'] == 'fr-CA' + assert collection['title'] == 'Observations' + assert collection['description'] == 'Mes belles observations' + + # Check HTML request in an unsupported language + req = mock_request({'f': 'html', 'lang': 'de'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'en-US' + + # hiearchical collections + req = mock_request() + rsp_headers, code, response = api_.describe_collections( + req, 'naturalearth/lakes') + + collection = json.loads(response) + assert collection['id'] == 'naturalearth/lakes' + + # OAPIF Part 2 CRS 6.2.1 B, defaults when not configured + assert collection['crs'] is not None + default_crs_list = [ + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', + 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', + ] + contains_default = False + for crs in default_crs_list: + if crs in default_crs_list: + contains_default = True + assert contains_default + assert collection['storageCRS'] is not None + assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa + assert collection['storageCrsCoordinateEpoch'] == 2017.23 + + +def test_describe_collections_hidden_resources( + config_hidden_resources, api_hidden_resources): + req = mock_request({}) + rsp_headers, code, response = api_hidden_resources.describe_collections(req) # noqa + assert code == HTTPStatus.OK + + assert len(config_hidden_resources['resources']) == 3 + + collections = json.loads(response) + assert len(collections['collections']) == 1 + + +def test_describe_collections_json_ld(config, api_): + req = mock_request({'f': 'jsonld'}) + rsp_headers, code, response = api_.describe_collections(req, 'obs') + collection = json.loads(response) + + assert '@context' in collection + expanded = jsonld.expand(collection)[0] + # Metadata is about a schema:DataCollection that contains a schema:Dataset + assert not expanded['@id'].endswith('obs') + assert 'http://schema.org/dataset' in expanded + assert len(expanded['http://schema.org/dataset']) == 1 + dataset = expanded['http://schema.org/dataset'][0] + assert dataset['@type'][0] == 'http://schema.org/Dataset' + assert len(dataset['http://schema.org/distribution']) == 14 + assert all(dist['@type'][0] == 'http://schema.org/DataDownload' + for dist in dataset['http://schema.org/distribution']) + + assert 'http://schema.org/Organization' in expanded[ + 'http://schema.org/provider'][0]['@type'] + + assert 'http://schema.org/Place' in dataset[ + 'http://schema.org/spatial'][0]['@type'] + assert 'http://schema.org/GeoShape' in dataset[ + 'http://schema.org/spatial'][0]['http://schema.org/geo'][0]['@type'] + assert dataset['http://schema.org/spatial'][0]['http://schema.org/geo'][ + 0]['http://schema.org/box'][0]['@value'] == '-180,-90 180,90' + + assert 'http://schema.org/temporalCoverage' in dataset + assert dataset['http://schema.org/temporalCoverage'][0][ + '@value'] == '2000-10-30T18:24:39+00:00/2007-10-30T08:57:29+00:00' + + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_describe_collections_enclosures(config_enclosure, enclosure_api): + original_enclosures = { + lnk['title']: lnk + for lnk in config_enclosure['resources']['objects']['links'] + if lnk['rel'] == 'enclosure' + } + + req = mock_request() + _, _, response = enclosure_api.describe_collections(req, 'objects') + features = json.loads(response) + modified_enclosures = { + lnk['title']: lnk for lnk in features['links'] + if lnk['rel'] == 'enclosure' + } + + # If type and length is set, do not verify/update link + assert original_enclosures['download link 1'] == \ + modified_enclosures['download link 1'] + # If length is missing, modify link type and length + assert original_enclosures['download link 2']['type'] == \ + modified_enclosures['download link 2']['type'] + assert modified_enclosures['download link 2']['type'] == \ + modified_enclosures['download link 3']['type'] + assert 'length' not in original_enclosures['download link 2'] + assert modified_enclosures['download link 2']['length'] > 0 + assert modified_enclosures['download link 2']['length'] == \ + modified_enclosures['download link 3']['length'] + assert original_enclosures['download link 3']['type'] != \ + modified_enclosures['download link 3']['type'] + + +def test_get_collection_schema(config, api_): + req = mock_request() + rsp_headers, code, response = api_.get_collection_schema(req, 'notfound') + assert code == HTTPStatus.NOT_FOUND + + req = mock_request({'f': 'html'}) + rsp_headers, code, response = api_.get_collection_schema(req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_request({'f': 'json'}) + rsp_headers, code, response = api_.get_collection_schema(req, 'obs') + assert rsp_headers['Content-Type'] == 'application/schema+json' + schema = json.loads(response) + + assert 'properties' in schema + assert len(schema['properties']) == 5 + + +def test_validate_bbox(): + assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] + assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] + assert validate_bbox('-142,42,-52,84') == [-142, 42, -52, 84] + assert (validate_bbox('-142.1,42.12,-52.22,84.4') == + [-142.1, 42.12, -52.22, 84.4]) + assert (validate_bbox('-142.1,42.12,-5.28,-52.22,84.4,7.39') == + [-142.1, 42.12, -5.28, -52.22, 84.4, 7.39]) + + assert (validate_bbox('177.0,65.0,-177.0,70.0') == + [177.0, 65.0, -177.0, 70.0]) + + with pytest.raises(ValueError): + validate_bbox('1,2,4') + + with pytest.raises(ValueError): + validate_bbox('1,2,4,5,6') + + with pytest.raises(ValueError): + validate_bbox('3,4,1,2') + + with pytest.raises(ValueError): + validate_bbox('1,2,6,4,5,3') + + +def test_validate_datetime(): + config = yaml_load(''' + temporal: + begin: 2000-10-30T18:24:39Z + end: 2007-10-30T08:57:29Z + ''') + + # test time instant + assert validate_datetime(config, '2004') == '2004' + assert validate_datetime(config, '2004-10') == '2004-10' + assert validate_datetime(config, '2001-10-30') == '2001-10-30' + + with pytest.raises(ValueError): + _ = validate_datetime(config, '2009-10-30') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2000-09-09') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2000-10-30T17:24:39Z') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2007-10-30T08:58:29Z') + + # test time envelope + assert validate_datetime(config, '2004/2005') == '2004/2005' + assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' + assert (validate_datetime(config, '2001-10-30/2002-10-30') == + '2001-10-30/2002-10-30') + assert validate_datetime(config, '2004/..') == '2004/..' + assert validate_datetime(config, '../2005') == '../2005' + assert validate_datetime(config, '2004/') == '2004/..' + assert validate_datetime(config, '/2005') == '../2005' + assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' + assert (validate_datetime(config, '2001-10-30/2002-10-30') == + '2001-10-30/2002-10-30') + + with pytest.raises(ValueError): + _ = validate_datetime(config, '2007-11-01/..') + with pytest.raises(ValueError): + _ = validate_datetime(config, '2009/..') + with pytest.raises(ValueError): + _ = validate_datetime(config, '../2000-09') + with pytest.raises(ValueError): + _ = validate_datetime(config, '../1999') + + +@pytest.mark.parametrize("value, expected", [ + ('time(2000-11-11)', {'time': ['2000-11-11']}), + ('time("2000-11-11")', {'time': ['2000-11-11']}), + ('time("2000-11-11T00:11:11")', {'time': ['2000-11-11T00:11:11']}), + ('time("2000-11-11T11:12:13":"2021-12-22T:13:33:33")', {'time': ['2000-11-11T11:12:13', '2021-12-22T:13:33:33']}), # noqa + ('lat(40)', {'lat': [40]}), + ('lat(0:40)', {'lat': [0, 40]}), + ('foo("bar")', {'foo': ['bar']}), + ('foo("bar":"baz")', {'foo': ['bar', 'baz']}) +]) +def test_validate_subset(value, expected): + assert validate_subset(value) == expected + + with pytest.raises(ValueError): + validate_subset('foo("bar)') + + +def test_get_exception(config, api_): + d = api_.get_exception(500, {}, 'json', 'NoApplicableCode', 'oops') + assert d[0] == {} + assert d[1] == 500 + content = json.loads(d[2]) + assert content['code'] == 'NoApplicableCode' + assert content['description'] == 'oops' + + d = api_.get_exception(500, {}, 'html', 'NoApplicableCode', 'oops') diff --git a/tests/api/test_coverages.py b/tests/api/test_coverages.py new file mode 100644 index 0000000..39b2752 --- /dev/null +++ b/tests/api/test_coverages.py @@ -0,0 +1,177 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import json +from http import HTTPStatus + +import pytest + +from pygeoapi.api.coverages import get_collection_coverage +from pygeoapi.util import yaml_load + +from tests.util import get_test_file_path, mock_request, mock_api_request + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +def test_describe_collections(config, api_): + + req = mock_request() + rsp_headers, code, response = api_.describe_collections( + req, 'gdps-temperature') + + collection = json.loads(response) + + assert collection['id'] == 'gdps-temperature' + assert len(collection['links']) == 10 + assert collection['extent']['spatial']['grid'][0]['cellsCount'] == 2400 + assert collection['extent']['spatial']['grid'][0]['resolution'] == 0.15000000000000002 # noqa + assert collection['extent']['spatial']['grid'][1]['cellsCount'] == 1201 + assert collection['extent']['spatial']['grid'][1]['resolution'] == 0.15 + + +def test_get_collection_schema(config, api_): + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = api_.get_collection_schema( + req, 'gdps-temperature') + + assert rsp_headers['Content-Type'] == 'application/schema+json' + + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = api_.get_collection_schema( + req, 'gdps-temperature') + + assert rsp_headers['Content-Type'] == 'application/schema+json' + schema = json.loads(response) + + assert 'properties' in schema + assert len(schema['properties']) == 1 + + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = api_.get_collection_schema( + req, 'gdps-temperature') + assert rsp_headers['Content-Type'] == 'application/schema+json' + schema = json.loads(response) + + assert 'properties' in schema + assert len(schema['properties']) == 1 + assert schema['properties']['1']['type'] == 'number' + assert schema['properties']['1']['title'] == 'Temperature [C]' + + +def test_get_collection_coverage(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_coverage( + api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'properties': '12'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'subset': 'bad_axis(10:20)'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'f': 'blah'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.BAD_REQUEST + assert rsp_headers['Content-Type'] == 'text/html' + + req = mock_api_request(HTTP_ACCEPT='text/html') + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + # NOTE: This test used to assert the code to be 200 OK, + # but it requested HTML, which is not available, + # so it should be 400 Bad Request + assert code == HTTPStatus.BAD_REQUEST + assert rsp_headers['Content-Type'] == 'text/html' + + req = mock_api_request({'subset': 'Lat(5:10),Long(5:10)'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.OK + content = json.loads(response) + + assert content['domain']['axes']['x']['num'] == 35 + assert content['domain']['axes']['y']['num'] == 35 + assert 'TMP' in content['parameters'] + assert 'TMP' in content['ranges'] + assert content['ranges']['TMP']['axisNames'] == ['y', 'x'] + + req = mock_api_request({'bbox': '-79,45,-75,49'}) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.OK + content = json.loads(response) + + assert content['domain']['axes']['x']['start'] == -79.0 + assert content['domain']['axes']['x']['stop'] == -75.0 + assert content['domain']['axes']['y']['start'] == 49.0 + assert content['domain']['axes']['y']['stop'] == 45.0 + + req = mock_api_request({ + 'subset': 'Lat(5:10),Long(5:10)', + 'f': 'GRIB' + }) + rsp_headers, code, response = get_collection_coverage( + api_, req, 'gdps-temperature') + + assert code == HTTPStatus.OK + assert isinstance(response, bytes) + + req = mock_api_request(HTTP_ACCEPT='application/x-netcdf') + rsp_headers, code, response = get_collection_coverage( + api_, req, 'cmip5') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == 'application/x-netcdf' diff --git a/tests/api/test_environmental_data_retrieval.py b/tests/api/test_environmental_data_retrieval.py new file mode 100644 index 0000000..0de57f0 --- /dev/null +++ b/tests/api/test_environmental_data_retrieval.py @@ -0,0 +1,226 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import json +from http import HTTPStatus + +from pygeoapi.api.environmental_data_retrieval import get_collection_edr_query + +from tests.util import mock_api_request + + +def test_get_collection_edr_query(config, api_): + # edr resource + req = mock_api_request() + rsp_headers, code, response = api_.describe_collections(req, 'icoads-sst') + collection = json.loads(response) + parameter_names = list(collection['parameter_names'].keys()) + parameter_names.sort() + assert len(parameter_names) == 4 + assert parameter_names == ['AIRT', 'SST', 'UWND', 'VWND'] + + # no coords parameter + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # bad query type + req = mock_api_request({'coords': 'POINT(11 11)'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'corridor') + assert code == HTTPStatus.BAD_REQUEST + + # bad coords parameter + req = mock_api_request({'coords': 'gah'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # bad parameter_names parameter + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'parameter_names': 'bad' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # all parameters + req = mock_api_request({'coords': 'POINT(11 11)'}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + + axes = list(data['domain']['axes'].keys()) + axes.sort() + assert len(axes) == 3 + assert axes == ['TIME', 'x', 'y'] + + assert data['domain']['axes']['x']['start'] == 11.0 + assert data['domain']['axes']['x']['stop'] == 11.0 + assert data['domain']['axes']['y']['start'] == 11.0 + assert data['domain']['axes']['y']['stop'] == 11.0 + + parameters = list(data['parameters'].keys()) + parameters.sort() + assert len(parameters) == 4 + assert parameters == ['AIRT', 'SST', 'UWND', 'VWND'] + + # single parameter + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'parameter_names': 'SST' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + + assert len(data['parameters'].keys()) == 1 + assert list(data['parameters'].keys())[0] == 'SST' + + # Zulu time zone + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '2000-01-17T00:00:00Z/2000-06-16T23:00:00Z' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + # bounded date range + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '2000-01-17/2000-06-16' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + time_dict = data['domain']['axes']['TIME'] + + assert time_dict['start'] == '2000-02-15T16:29:05.999999999' + assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' + assert time_dict['num'] == 5 + + # unbounded date range - start + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '../2000-06-16' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + time_dict = data['domain']['axes']['TIME'] + + assert time_dict['start'] == '2000-01-16T06:00:00.000000000' + assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' + assert time_dict['num'] == 6 + + # unbounded date range - end + req = mock_api_request({ + 'coords': 'POINT(11 11)', + 'datetime': '2000-06-16/..' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + data = json.loads(response) + time_dict = data['domain']['axes']['TIME'] + + assert time_dict['start'] == '2000-06-16T10:25:30.000000000' + assert time_dict['stop'] == '2000-12-16T01:20:05.999999996' + assert time_dict['num'] == 7 + + # some data + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'datetime': '2000-01-16' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.OK + + # no data + req = mock_api_request({ + 'coords': 'POINT(11 11)', 'datetime': '2000-01-17' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.NO_CONTENT + + # position no coords + req = mock_api_request({ + 'datetime': '2000-01-17' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'position') + assert code == HTTPStatus.BAD_REQUEST + + # cube bbox parameter 4 dimensional + req = mock_api_request({ + 'bbox': '0,0,10,10' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') + assert code == HTTPStatus.OK + + # cube bad bbox parameter + req = mock_api_request({ + 'bbox': '0,0,10' + }) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') + assert code == HTTPStatus.BAD_REQUEST + + # cube no bbox parameter + req = mock_api_request({}) + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'icoads-sst', None, 'cube') + assert code == HTTPStatus.BAD_REQUEST + + # cube decreasing latitude coords and S3 + req = mock_api_request({ + 'bbox': '-100,40,-99,45', + 'parameter_names': 'tmn', + 'datetime': '1994-01-01/1994-12-31', + }) + + rsp_headers, code, response = get_collection_edr_query( + api_, req, 'usgs-prism', None, 'cube') + assert code == HTTPStatus.OK diff --git a/tests/api/test_itemtypes.py b/tests/api/test_itemtypes.py new file mode 100644 index 0000000..1ced8ea --- /dev/null +++ b/tests/api/test_itemtypes.py @@ -0,0 +1,659 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import copy +import gzip +import json +from http import HTTPStatus + +from pyld import jsonld +import pytest +import pyproj +from shapely.geometry import Point + +from pygeoapi.api import (API, FORMAT_TYPES, F_GZIP, F_HTML, F_JSONLD, + apply_gzip) +from pygeoapi.api.itemtypes import ( + get_collection_queryables, get_collection_item, + get_collection_items, manage_collection_item) +from pygeoapi.util import yaml_load, get_crs_from_uri + +from tests.util import get_test_file_path, mock_api_request + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +def test_get_collection_queryables(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_queryables( + api_, req, 'notfound') + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') + assert rsp_headers['Content-Type'] == 'application/schema+json' + queryables = json.loads(response) + + assert 'properties' in queryables + assert len(queryables['properties']) == 5 + + # test with provider filtered properties + api_.config['resources']['obs']['providers'][0]['properties'] = ['stn_id'] + + rsp_headers, code, response = get_collection_queryables(api_, req, 'obs') + queryables = json.loads(response) + + assert 'properties' in queryables + assert len(queryables['properties']) == 2 + assert 'geometry' in queryables['properties'] + assert queryables['properties']['geometry']['$ref'] == 'https://geojson.org/schema/Geometry.json' # noqa + + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_get_collection_items(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_items(api_, req, 'foo') + features = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request({'f': 'foo'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox': '1,2,3'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox': '1,2,3,4c'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox': '1,2,3,4', 'bbox-crs': 'bad_value'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'bbox-crs': 'bad_value'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + # bbox-crs must be in configured values for Collection + req = mock_api_request({'bbox': '1,2,3,4', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4258'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + # bbox-crs must be in configured values for Collection (CSV will ignore) + req = mock_api_request({'bbox': '52,4,53,5', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + # bbox-crs can be a default even if not configured + req = mock_api_request({'bbox': '4,52,5,53', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + # bbox-crs can be a default even if not configured + req = mock_api_request({'bbox': '4,52,5,53'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'f': 'html', 'lang': 'fr'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'fr-CA' + + req = mock_api_request() + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + assert len(features['features']) == 5 + + req = mock_api_request({'resulttype': 'hits'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 0 + + # Invalid limit + req = mock_api_request({'limit': 0}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'stn_id': '35'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 2 + assert features['numberMatched'] == 2 + + req = mock_api_request({'stn_id': '35', 'value': '93.9'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 1 + assert features['numberMatched'] == 1 + + req = mock_api_request({'limit': 2}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 2 + assert features['features'][1]['properties']['stn_id'] == 35 + + links = features['links'] + assert len(links) == 4 + assert '/collections/obs/items?f=json' in links[0]['href'] + assert links[0]['rel'] == 'self' + assert '/collections/obs/items?f=jsonld' in links[1]['href'] + assert links[1]['rel'] == 'alternate' + assert '/collections/obs/items?f=html' in links[2]['href'] + assert links[2]['rel'] == 'alternate' + assert '/collections/obs' in links[3]['href'] + assert links[3]['rel'] == 'collection' + + # Invalid offset + req = mock_api_request({'offset': -1}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'offset': 2}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 3 + assert features['features'][1]['properties']['stn_id'] == 2147 + + links = features['links'] + assert len(links) == 5 + assert '/collections/obs/items?f=json' in links[0]['href'] + assert links[0]['rel'] == 'self' + assert '/collections/obs/items?f=jsonld' in links[1]['href'] + assert links[1]['rel'] == 'alternate' + assert '/collections/obs/items?f=html' in links[2]['href'] + assert links[2]['rel'] == 'alternate' + assert '/collections/obs/items?offset=0' in links[3]['href'] + assert links[3]['rel'] == 'prev' + assert '/collections/obs' in links[4]['href'] + assert links[4]['rel'] == 'collection' + + req = mock_api_request({ + 'offset': 1, + 'limit': 1, + 'bbox': '-180,90,180,90' + }) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + + assert len(features['features']) == 1 + + links = features['links'] + assert len(links) == 5 + assert '/collections/obs/items?f=json&limit=1&bbox=-180,90,180,90' in \ + links[0]['href'] + assert links[0]['rel'] == 'self' + assert '/collections/obs/items?f=jsonld&limit=1&bbox=-180,90,180,90' in \ + links[1]['href'] + assert links[1]['rel'] == 'alternate' + assert '/collections/obs/items?f=html&limit=1&bbox=-180,90,180,90' in \ + links[2]['href'] + assert links[2]['rel'] == 'alternate' + assert '/collections/obs/items?offset=0&limit=1&bbox=-180,90,180,90' \ + in links[3]['href'] + assert links[3]['rel'] == 'prev' + assert '/collections/obs' in links[4]['href'] + assert links[4]['rel'] == 'collection' + + req = mock_api_request({ + 'sortby': 'bad-property', + 'stn_id': '35' + }) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'sortby': 'stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + assert code == HTTPStatus.OK + + req = mock_api_request({'sortby': '+stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + assert code == HTTPStatus.OK + + req = mock_api_request({'sortby': '-stn_id'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + features = json.loads(response) + assert code == HTTPStatus.OK + + req = mock_api_request({'f': 'csv'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert rsp_headers['Content-Type'] == 'text/csv; charset=utf-8' + + req = mock_api_request({'datetime': '2003'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '1999'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'datetime': '2010-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'datetime': '2001-11-11/2003-12-18'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '../2003-12-18'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '2001-11-11/..'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '1999/2005-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'datetime': '1999/2000-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + api_.config['resources']['obs']['extents'].pop('temporal') + + req = mock_api_request({'datetime': '2002/2014-04-22'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.OK + + req = mock_api_request({'scalerank': 1}) + rsp_headers, code, response = get_collection_items( + api_, req, 'naturalearth/lakes') + features = json.loads(response) + + assert len(features['features']) == 10 + assert features['numberMatched'] == 11 + assert features['numberReturned'] == 10 + + req = mock_api_request({'datetime': '2005-04-22'}) + rsp_headers, code, response = get_collection_items( + api_, req, 'naturalearth/lakes') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'skipGeometry': 'true'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert json.loads(response)['features'][0]['geometry'] is None + + req = mock_api_request({'properties': 'foo,bar'}) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert code == HTTPStatus.BAD_REQUEST + + +def test_collection_items_gzip_csv(config, api_, openapi): + # Add gzip to server + config['server']['gzip'] = True + api_ = API(config, openapi) + + req_csv = mock_api_request({'f': 'csv'}) + rsp_csv_headers, _, rsp_csv = get_collection_items(api_, req_csv, 'obs') + rsp_csv = apply_gzip(rsp_csv_headers, rsp_csv) + assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' + rsp_csv = rsp_csv.decode('utf-8') + + req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) + rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa + rsp_csv_gzip = apply_gzip(rsp_csv_headers, rsp_csv_gzip) + assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' + rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') + assert rsp_csv == rsp_csv_ + + # Use utf-16 encoding + config['server']['encoding'] = 'utf-16' + api_ = API(config, openapi) + + req_csv = mock_api_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) + rsp_csv_headers, _, rsp_csv_gzip = get_collection_items(api_, req_csv, 'obs') # noqa + rsp_csv_gzip = apply_gzip(rsp_csv_headers, rsp_csv_gzip) + assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' + rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') + assert rsp_csv == rsp_csv_ + + +def test_get_collection_items_crs(config, api_): + + # Invalid CRS query parameter + req = mock_api_request({'crs': '4326'}) + rsp_headers, code, response = get_collection_items(api_, req, 'norway_pop') + + assert code == HTTPStatus.BAD_REQUEST + + # Unsupported CRS + req = mock_api_request( + {'crs': 'http://www.opengis.net/def/crs/EPSG/0/32633'}) + rsp_headers, code, response = get_collection_items(api_, req, 'norway_pop') + + assert code == HTTPStatus.BAD_REQUEST + + # Supported CRSs + default_crs = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + storage_crs = 'http://www.opengis.net/def/crs/EPSG/0/25833' + crs_4258 = 'http://www.opengis.net/def/crs/EPSG/0/4258' + supported_crs_list = [default_crs, storage_crs, crs_4258] + + for crs in supported_crs_list: + req = mock_api_request({'crs': crs}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{crs}>' + + # With CRS query parameter, using storageCRS + req = mock_api_request({'crs': storage_crs}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' + + features_25833 = json.loads(response) + + # With CRS query parameter resulting in coordinates transformation + req = mock_api_request({'crs': crs_4258}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{crs_4258}>' + + features_4258 = json.loads(response) + transform_func = pyproj.Transformer.from_crs( + pyproj.CRS.from_epsg(25833), + pyproj.CRS.from_epsg(4258), + always_xy=False, + ).transform + for feat_orig in features_25833['features']: + id_ = feat_orig['id'] + x, y, *_ = feat_orig['geometry']['coordinates'] + loc_transf = Point(transform_func(x, y)) + for feat_out in features_4258['features']: + if id_ == feat_out['id']: + loc_out = Point(feat_out['geometry']['coordinates'][:2]) + + assert loc_out.equals_exact(loc_transf, 1e-5) + break + + # Without CRS query parameter: assume Transform to default WGS84 lon,lat + req = mock_api_request({}) + rsp_headers, code, response = get_collection_items( + api_, req, 'norway_pop') + + assert code == HTTPStatus.OK + assert rsp_headers['Content-Crs'] == f'<{default_crs}>' + + features_wgs84 = json.loads(response) + + # With CRS query parameter resulting in coordinates transformation + transform_func = pyproj.Transformer.from_crs( + pyproj.CRS.from_epsg(4258), + get_crs_from_uri(default_crs), + always_xy=False, + ).transform + for feat_orig in features_4258['features']: + id_ = feat_orig['id'] + x, y, *_ = feat_orig['geometry']['coordinates'] + loc_transf = Point(transform_func(x, y)) + for feat_out in features_wgs84['features']: + if id_ == feat_out['id']: + loc_out = Point(feat_out['geometry']['coordinates'][:2]) + + assert loc_out.equals_exact(loc_transf, 1e-5) + break + + +def test_manage_collection_item_read_only_options_req(config, api_): + """Test OPTIONS request on a read-only items endpoint""" + req = mock_api_request() + _, code, _ = manage_collection_item(api_, req, 'options', 'foo') + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item(api_, req, 'options', 'obs') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET' + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item( + api_, req, 'options', 'obs', 'ressource_id') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET' + + +def test_manage_collection_item_editable_options_req(config, openapi): + """Test OPTIONS request on a editable items endpoint""" + config = copy.deepcopy(config) + config['resources']['obs']['providers'][0]['editable'] = True + api_ = API(config, openapi) + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item(api_, req, 'options', 'obs') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET, POST' + + req = mock_api_request() + rsp_headers, code, _ = manage_collection_item( + api_, req, 'options', 'obs', 'ressource_id') + assert code == HTTPStatus.OK + assert rsp_headers['Allow'] == 'HEAD, GET, PUT, DELETE' + + +def test_get_collection_items_json_ld(config, api_): + req = mock_api_request({ + 'f': 'jsonld', + 'limit': 2 + }) + rsp_headers, code, response = get_collection_items(api_, req, 'obs') + + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + # No language requested: return default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + collection = json.loads(response) + + assert '@context' in collection + assert all((f in collection['@context'][0] for + f in ('schema', 'type', 'features', 'FeatureCollection'))) + assert len(collection['@context']) > 1 + assert collection['@context'][1]['schema'] == 'https://schema.org/' + expanded = jsonld.expand(collection)[0] + featuresUri = 'https://schema.org/itemListElement' + assert len(expanded[featuresUri]) == 2 + + +def test_get_collection_item(config, api_): + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_item( + api_, req, 'gdps-temperature', '371') + + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request() + rsp_headers, code, response = get_collection_item(api_, req, 'foo', '371') + + assert code == HTTPStatus.NOT_FOUND + + rsp_headers, code, response = get_collection_item( + api_, req, 'obs', 'notfound') + + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') + + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + assert rsp_headers['Content-Language'] == 'en-US' + + req = mock_api_request() + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') + feature = json.loads(response) + + assert feature['properties']['stn_id'] == 35 + assert 'prev' not in feature['links'] + assert 'next' not in feature['links'] + + +def test_get_collection_item_json_ld(config, api_): + req = mock_api_request({'f': 'jsonld'}) + rsp_headers, _, response = get_collection_item(api_, req, 'objects', '3') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + assert rsp_headers['Content-Language'] == 'en-US' + feature = json.loads(response) + assert '@context' in feature + assert all((f in feature['@context'][0] for + f in ('schema', 'type', 'gsp'))) + assert len(feature['@context']) == 1 + assert 'schema' in feature['@context'][0] + assert feature['@context'][0]['schema'] == 'https://schema.org/' + assert feature['id'] == 3 + expanded = jsonld.expand(feature)[0] + + assert expanded['@id'].startswith('http://') + assert expanded['@id'].endswith('/collections/objects/items/3') + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'POINT (-85 33)' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/latitude'][0][ + '@value'] == 33 + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/longitude'][0][ + '@value'] == -85 + + _, _, response = get_collection_item(api_, req, 'objects', '2') + feature = json.loads(response) + assert feature['geometry']['type'] == 'MultiPoint' + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'MULTIPOINT (10 40, 40 30, 20 20, 30 10)' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/polygon'][0][ + '@value'] == "10.0,40.0 40.0,30.0 20.0,20.0 30.0,10.0 10.0,40.0" + + _, _, response = get_collection_item(api_, req, 'objects', '1') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'LINESTRING (30 10, 10 30, 40 40)' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/line'][0][ + '@value'] == '30.0,10.0 10.0,30.0 40.0,40.0' + + _, _, response = get_collection_item(api_, req, 'objects', '4') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'MULTILINESTRING ((10 10, 20 20, 10 40), ' \ + '(40 40, 30 30, 40 20, 30 10))' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/line'][0][ + '@value'] == '10.0,10.0 20.0,20.0 10.0,40.0 40.0,40.0 ' \ + '30.0,30.0 40.0,20.0 30.0,10.0' + + _, _, response = get_collection_item(api_, req, 'objects', '5') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/polygon'][0][ + '@value'] == '30.0,10.0 40.0,40.0 20.0,40.0 10.0,20.0 30.0,10.0' + + _, _, response = get_collection_item(api_, req, 'objects', '7') + feature = json.loads(response) + expanded = jsonld.expand(feature)[0] + assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ + 'http://www.opengis.net/ont/geosparql#asWKT'][0][ + '@value'] == 'MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), '\ + '((15 5, 40 10, 10 20, 5 10, 15 5)))' + assert expanded['https://schema.org/geo'][0][ + 'https://schema.org/polygon'][0][ + '@value'] == '15.0,5.0 5.0,10.0 10.0,40.0 '\ + '45.0,40.0 40.0,10.0 15.0,5.0' + + req = mock_api_request({'f': 'jsonld', 'lang': 'fr'}) + rsp_headers, code, response = get_collection_item(api_, req, 'obs', '371') + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] + assert rsp_headers['Content-Language'] == 'fr-CA' diff --git a/tests/api/test_maps.py b/tests/api/test_maps.py new file mode 100644 index 0000000..3b284ac --- /dev/null +++ b/tests/api/test_maps.py @@ -0,0 +1,52 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +from http import HTTPStatus + +from pygeoapi.api.maps import get_collection_map + +from tests.util import mock_api_request + + +def test_get_collection_map(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_map(api_, req, 'notfound') + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request() + rsp_headers, code, response = get_collection_map( + api_, req, 'mapserver_world_map') + assert code == HTTPStatus.OK + assert isinstance(response, bytes) + assert response[1:4] == b'PNG' diff --git a/tests/api/test_processes.py b/tests/api/test_processes.py new file mode 100644 index 0000000..b83305f --- /dev/null +++ b/tests/api/test_processes.py @@ -0,0 +1,428 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import json +from http import HTTPStatus +import time +from unittest import mock + +from pygeoapi.api import FORMAT_TYPES, F_HTML, F_JSON +from pygeoapi.api.processes import ( + describe_processes, execute_process, delete_job, get_job_result, +) + +from tests.util import mock_api_request + + +def test_describe_processes(config, api_): + req = mock_api_request({'limit': 1}) + # Test for description of single processes + rsp_headers, code, response = describe_processes(api_, req) + data = json.loads(response) + assert code == HTTPStatus.OK + assert len(data['processes']) == 1 + assert len(data['links']) == 3 + + req = mock_api_request() + + # Test for undefined process + rsp_headers, code, response = describe_processes(api_, req, 'foo') + data = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert data['code'] == 'NoSuchProcess' + + # Test for description of all processes + rsp_headers, code, response = describe_processes(api_, req) + data = json.loads(response) + assert code == HTTPStatus.OK + assert len(data['processes']) == 2 + assert len(data['links']) == 3 + + # Test for particular, defined process + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + process = json.loads(response) + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert process['id'] == 'hello-world' + assert process['version'] == '0.2.0' + assert process['title'] == 'Hello World' + assert len(process['keywords']) == 3 + assert len(process['links']) == 6 + assert len(process['inputs']) == 2 + assert len(process['outputs']) == 1 + assert len(process['outputTransmission']) == 1 + assert len(process['jobControlOptions']) == 2 + assert 'sync-execute' in process['jobControlOptions'] + assert 'async-execute' in process['jobControlOptions'] + + # Check HTML response when requested in headers + req = mock_api_request(HTTP_ACCEPT='text/html') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: return default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + # Check JSON response when requested in headers + req = mock_api_request(HTTP_ACCEPT='application/json') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'en-US' + + # Check HTML response when requested with query parameter + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: return default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + # Check JSON response when requested with query parameter + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'en-US' + + # Check JSON response when requested with French language parameter + req = mock_api_request({'lang': 'fr'}) + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'fr-CA' + process = json.loads(response) + assert process['title'] == 'Bonjour le Monde' + + # Check JSON response when language requested in headers + req = mock_api_request(HTTP_ACCEPT_LANGUAGE='fr') + rsp_headers, code, response = describe_processes(api_, req, 'hello-world') + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + assert rsp_headers['Content-Language'] == 'fr-CA' + + # Test for undefined process + req = mock_api_request() + rsp_headers, code, response = describe_processes(api_, req, + 'goodbye-world') + data = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert data['code'] == 'NoSuchProcess' + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] + + # Test describe doesn't crash if example is missing + req = mock_api_request() + processor = api_.manager.get_processor("hello-world") + example = processor.metadata.pop("example") + rsp_headers, code, response = describe_processes(api_, req) + processor.metadata['example'] = example + data = json.loads(response) + assert code == HTTPStatus.OK + assert len(data['processes']) == 2 + + +def test_execute_process(config, api_): + req_body_0 = { + 'inputs': { + 'name': 'Test' + } + } + req_body_1 = { + 'inputs': { + 'name': 'Test' + }, + 'response': 'document' + } + req_body_2 = { + 'inputs': { + 'name': 'Tést' + } + } + req_body_3 = { + 'inputs': { + 'name': 'Tést', + 'message': 'This is a test.' + } + } + req_body_4 = { + 'inputs': { + 'foo': 'Tést' + } + } + req_body_5 = { + 'inputs': {} + } + req_body_6 = { + 'inputs': { + 'name': None + } + } + req_body_7 = { + 'inputs': { + 'name': 'Test' + }, + 'subscriber': { + 'successUri': 'https://example.com/success', + 'inProgressUri': 'https://example.com/inProgress', + 'failedUri': 'https://example.com/failed', + } + } + + cleanup_jobs = set() + + # Test posting empty payload to existing process + req = mock_api_request(data='') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + assert rsp_headers['Content-Language'] == 'en-US' + + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' not in rsp_headers + assert data['code'] == 'MissingParameterValue' + + req = mock_api_request(data=req_body_0) + rsp_headers, code, response = execute_process(api_, req, 'foo') + + data = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert 'Location' not in rsp_headers + assert data['code'] == 'NoSuchProcess' + + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + + assert len(data.keys()) == 2 + assert data['id'] == 'echo' + assert data['value'] == 'Hello Test!' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_1) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + + assert len(data.keys()) == 1 + assert data['outputs'][0]['id'] == 'echo' + assert data['outputs'][0]['value'] == 'Hello Test!' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_2) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + assert data['value'] == 'Hello Tést!' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_3) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + assert data['value'] == 'Hello Tést! This is a test.' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_4) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' in rsp_headers + assert data['code'] == 'InvalidParameterValue' + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_5) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' in rsp_headers + assert data['code'] == 'InvalidParameterValue' + assert data['description'] == 'Error updating job' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_6) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.BAD_REQUEST + assert 'Location' in rsp_headers + assert data['code'] == 'InvalidParameterValue' + assert data['description'] == 'Error updating job' + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_0) + rsp_headers, code, response = execute_process(api_, req, 'goodbye-world') + + response = json.loads(response) + assert code == HTTPStatus.NOT_FOUND + assert 'Location' not in rsp_headers + assert response['code'] == 'NoSuchProcess' + + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + response = json.loads(response) + assert code == HTTPStatus.OK + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_1, HTTP_Prefer='respond-async') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + assert 'Location' in rsp_headers + response = json.loads(response) + assert isinstance(response, dict) + assert code == HTTPStatus.CREATED + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + req = mock_api_request(data=req_body_7) + with mock.patch( + 'pygeoapi.process.manager.base.requests.post' + ) as post_mocker: + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + assert code == HTTPStatus.OK + post_mocker.assert_any_call( + req_body_7['subscriber']['inProgressUri'], json={} + ) + post_mocker.assert_any_call( + req_body_7['subscriber']['successUri'], + json={'id': 'echo', 'value': 'Hello Test!'} + ) + assert post_mocker.call_count == 2 + + cleanup_jobs.add(tuple(['hello-world', + rsp_headers['Location'].split('/')[-1]])) + + # Cleanup + time.sleep(2) # Allow time for any outstanding async jobs + for _, job_id in cleanup_jobs: + rsp_headers, code, response = delete_job(api_, mock_api_request(), + job_id) + assert code == HTTPStatus.OK + + +def _execute_a_job(api_): + req_body_sync = { + 'inputs': { + 'name': 'Sync Test' + } + } + + req = mock_api_request(data=req_body_sync) + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + data = json.loads(response) + assert code == HTTPStatus.OK + assert 'Location' in rsp_headers + assert data['value'] == 'Hello Sync Test!' + + job_id = rsp_headers['Location'].split('/')[-1] + return job_id + + +def test_delete_job(api_): + rsp_headers, code, response = delete_job(api_, mock_api_request(), + 'does-not-exist') + + assert code == HTTPStatus.NOT_FOUND + req_body_async = { + 'inputs': { + 'name': 'Async Test Deletion' + } + } + job_id = _execute_a_job(api_) + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + + assert code == HTTPStatus.OK + + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + assert code == HTTPStatus.NOT_FOUND + + req = mock_api_request(data=req_body_async, HTTP_Prefer='respond-async') + rsp_headers, code, response = execute_process(api_, req, 'hello-world') + + assert code == HTTPStatus.CREATED + assert 'Location' in rsp_headers + + time.sleep(2) # Allow time for async execution to complete + job_id = rsp_headers['Location'].split('/')[-1] + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + assert code == HTTPStatus.OK + + rsp_headers, code, response = delete_job(api_, mock_api_request(), job_id) + assert code == HTTPStatus.NOT_FOUND + + +def test_get_job_result(api_): + rsp_headers, code, response = get_job_result( + api_, mock_api_request(), 'not-exist', + ) + assert code == HTTPStatus.NOT_FOUND + + job_id = _execute_a_job(api_) + rsp_headers, code, response = get_job_result(api_, mock_api_request(), + job_id) + # default response is html + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == 'text/html' + assert 'Hello Sync Test!' in response + + rsp_headers, code, response = get_job_result( + api_, mock_api_request({'f': 'json'}), job_id, + ) + assert code == HTTPStatus.OK + assert rsp_headers['Content-Type'] == 'application/json' + assert json.loads(response)['value'] == "Hello Sync Test!" diff --git a/tests/api/test_tiles.py b/tests/api/test_tiles.py new file mode 100644 index 0000000..df11ca5 --- /dev/null +++ b/tests/api/test_tiles.py @@ -0,0 +1,110 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# John A Stevenson +# Colin Blackburn +# Bernhard Mallinger +# +# Copyright (c) 2024 Tom Kralidis +# Copyright (c) 2022 John A Stevenson and Colin Blackburn +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +import json +from http import HTTPStatus + +from pygeoapi.api import FORMAT_TYPES, F_HTML +from pygeoapi.api.tiles import ( + get_collection_tiles, tilematrixset, tilematrixsets, +) +from pygeoapi.models.provider.base import TileMatrixSetEnum + +from tests.util import mock_api_request + + +def test_get_collection_tiles(config, api_): + req = mock_api_request() + rsp_headers, code, response = get_collection_tiles(api_, req, 'obs') + assert code == HTTPStatus.BAD_REQUEST + + rsp_headers, code, response = get_collection_tiles( + api_, req, 'naturalearth/lakes') + assert code == HTTPStatus.OK + + # Language settings should be ignored (return system default) + req = mock_api_request({'lang': 'fr'}) + rsp_headers, code, response = get_collection_tiles( + api_, req, 'naturalearth/lakes') + assert rsp_headers['Content-Language'] == 'en-US' + content = json.loads(response) + assert len(content['links']) > 0 + assert len(content['tilesets']) > 0 + + +def test_tilematrixsets(config, api_): + req = mock_api_request() + rsp_headers, code, response = tilematrixsets(api_, req) + root = json.loads(response) + + assert isinstance(root, dict) + assert 'tileMatrixSets' in root + assert len(root['tileMatrixSets']) == 2 + assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldCRS84Quad' \ + in root['tileMatrixSets'][0]['uri'] + assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad' \ + in root['tileMatrixSets'][1]['uri'] + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = tilematrixsets(api_, req) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' + + +def test_tilematrixset(config, api_): + req = mock_api_request() + + enums = [e.value for e in TileMatrixSetEnum] + enum = None + + for e in enums: + enum = e.tileMatrixSet + rsp_headers, code, response = tilematrixset(api_, req, enum) + root = json.loads(response) + + assert isinstance(root, dict) + assert 'id' in root + assert root['id'] == enum + assert 'tileMatrices' in root + assert len(root['tileMatrices']) == 30 + + rsp_headers, code, response = tilematrixset(api_, req, 'foo') + assert code == HTTPStatus.BAD_REQUEST + + req = mock_api_request({'f': 'html'}) + rsp_headers, code, response = tilematrixset(api_, req, enum) + assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] + # No language requested: should be set to default from YAML + assert rsp_headers['Content-Language'] == 'en-US' diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..ddeb3b4 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,52 @@ +# ================================================================= +# +# Authors: Bernhard Mallinger +# +# Copyright (c) 2024 Bernhard Mallinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import pytest + +from pygeoapi.api import API +from pygeoapi.util import yaml_load + +from tests.util import get_test_file_path + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def openapi(): + with open(get_test_file_path('pygeoapi-test-openapi.yml')) as fh: + return yaml_load(fh) + + +@pytest.fixture() +def api_(config, openapi): + return API(config, openapi) diff --git a/tests/test_api.py b/tests/test_api.py deleted file mode 100644 index 3c13af4..0000000 --- a/tests/test_api.py +++ /dev/null @@ -1,2262 +0,0 @@ -# ================================================================= -# -# Authors: Tom Kralidis -# John A Stevenson -# Colin Blackburn -# -# Copyright (c) 2023 Tom Kralidis -# Copyright (c) 2022 John A Stevenson and Colin Blackburn -# -# Permission is hereby granted, free of charge, to any person -# obtaining a copy of this software and associated documentation -# files (the "Software"), to deal in the Software without -# restriction, including without limitation the rights to use, -# copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following -# conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -# ================================================================= - -import copy -import json -import logging -import time -import gzip -from http import HTTPStatus -from unittest import mock - -from pyld import jsonld -import pytest -import pyproj -from shapely.geometry import Point - -from pygeoapi.api import ( - API, APIRequest, FORMAT_TYPES, validate_bbox, validate_datetime, - validate_subset, F_HTML, F_JSON, F_JSONLD, F_GZIP, __version__ -) -from pygeoapi.util import (yaml_load, get_crs_from_uri, - get_api_rules, get_base_url) - -from .util import (get_test_file_path, mock_request, - mock_flask, mock_starlette) - -from pygeoapi.models.provider.base import TileMatrixSetEnum - -LOGGER = logging.getLogger(__name__) - - -@pytest.fixture() -def config(): - with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_with_rules() -> dict: - """ Returns a pygeoapi configuration with default API rules. """ - with open(get_test_file_path('pygeoapi-test-config-apirules.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_enclosure() -> dict: - """ Returns a pygeoapi configuration with enclosure links. """ - with open(get_test_file_path('pygeoapi-test-config-enclosure.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def config_hidden_resources(): - filename = 'pygeoapi-test-config-hidden-resources.yml' - with open(get_test_file_path(filename)) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def openapi(): - with open(get_test_file_path('pygeoapi-test-openapi.yml')) as fh: - return yaml_load(fh) - - -@pytest.fixture() -def api_(config, openapi): - return API(config, openapi) - - -@pytest.fixture() -def enclosure_api(config_enclosure, openapi): - """ Returns an API instance with a collection with enclosure links. """ - return API(config_enclosure, openapi) - - -@pytest.fixture() -def rules_api(config_with_rules, openapi): - """ Returns an API instance with URL prefix and strict slashes policy. - The API version is extracted from the current version here. - """ - return API(config_with_rules, openapi) - - -@pytest.fixture() -def api_hidden_resources(config_hidden_resources, openapi): - return API(config_hidden_resources, openapi) - - -def test_apirequest(api_): - # Test without (valid) locales - with pytest.raises(ValueError): - req = mock_request() - APIRequest(req, []) - APIRequest(req, None) - APIRequest(req, ['zz']) - - # Test all supported formats from query args - for f, mt in FORMAT_TYPES.items(): - req = mock_request({'f': f}) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == f - assert apireq.get_response_headers()['Content-Type'] == mt - - # Test all supported formats from Accept header - for f, mt in FORMAT_TYPES.items(): - req = mock_request(HTTP_ACCEPT=mt) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == f - assert apireq.get_response_headers()['Content-Type'] == mt - - # Test nonsense format - req = mock_request({'f': 'foo'}) - apireq = APIRequest(req, api_.locales) - assert not apireq.is_valid() - assert apireq.format == 'foo' - assert apireq.is_valid(('foo',)) - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSON] - - # Test without format - req = mock_request() - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format is None - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSON] - assert apireq.get_linkrel(F_JSON) == 'self' - assert apireq.get_linkrel(F_HTML) == 'alternate' - - # Test complex format string - hh = 'text/html,application/xhtml+xml,application/xml;q=0.9,' - req = mock_request(HTTP_ACCEPT=hh) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_HTML - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_HTML] - assert apireq.get_linkrel(F_HTML) == 'self' - assert apireq.get_linkrel(F_JSON) == 'alternate' - - # Test accept header with multiple valid formats - hh = 'plain/text,application/ld+json,application/json;q=0.9,' - req = mock_request(HTTP_ACCEPT=hh) - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_JSONLD - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_JSONLD] - assert apireq.get_linkrel(F_JSONLD) == 'self' - assert apireq.get_linkrel(F_HTML) == 'alternate' - - # Overrule HTTP content negotiation - req = mock_request({'f': 'html'}, HTTP_ACCEPT='application/json') # noqa - apireq = APIRequest(req, api_.locales) - assert apireq.is_valid() - assert apireq.format == F_HTML - assert apireq.get_response_headers()['Content-Type'] == \ - FORMAT_TYPES[F_HTML] - - # Test data - for d in (None, '', 'test', {'key': 'value'}): - req = mock_request(data=d) - apireq = APIRequest.with_data(req, api_.locales) - if not d: - assert apireq.data == b'' - elif isinstance(d, dict): - assert d == json.loads(apireq.data) - else: - assert apireq.data == d.encode() - - # Test multilingual - test_lang = { - 'nl': ('en', 'en-US'), # unsupported lang should return default - 'en-US': ('en', 'en-US'), - 'de_CH': ('en', 'en-US'), - 'fr-CH, fr;q=0.9, en;q=0.8': ('fr', 'fr-CA'), - 'fr-CH, fr-BE;q=0.9': ('fr', 'fr-CA'), - } - sup_lang = ('en-US', 'fr_CA') - for lang_in, (lang_out, cl_out) in test_lang.items(): - # Using l query parameter - req = mock_request({'lang': lang_in}) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == lang_in - assert apireq.locale.language == lang_out - assert apireq.get_response_headers()['Content-Language'] == cl_out - - # Using Accept-Language header - req = mock_request(HTTP_ACCEPT_LANGUAGE=lang_in) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == lang_in - assert apireq.locale.language == lang_out - assert apireq.get_response_headers()['Content-Language'] == cl_out - - # Test language override - req = mock_request({'lang': 'fr'}, HTTP_ACCEPT_LANGUAGE='en_US') - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == 'fr' - assert apireq.locale.language == 'fr' - assert apireq.get_response_headers()['Content-Language'] == 'fr-CA' - - # Test locale territory - req = mock_request({'lang': 'en-GB'}) - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale == 'en-GB' - assert apireq.locale.language == 'en' - assert apireq.locale.territory == 'US' - assert apireq.get_response_headers()['Content-Language'] == 'en-US' - - # Test without Accept-Language header or 'lang' query parameter - # (should return default language from YAML config) - req = mock_request() - apireq = APIRequest(req, api_.locales) - assert apireq.raw_locale is None - assert apireq.locale.language == api_.default_locale.language - assert apireq.get_response_headers()['Content-Language'] == 'en-US' - - # Test without Accept-Language header or 'lang' query param - # (should return first in custom list of languages) - sup_lang = ('de', 'fr', 'en') - apireq = APIRequest(req, sup_lang) - assert apireq.raw_locale is None - assert apireq.locale.language == 'de' - assert apireq.get_response_headers()['Content-Language'] == 'de' - - -def test_apirules_active(config_with_rules, rules_api): - assert rules_api.config == config_with_rules - rules = get_api_rules(config_with_rules) - base_url = get_base_url(config_with_rules) - - # Test Flask - flask_prefix = rules.get_url_prefix('flask') - with mock_flask('pygeoapi-test-config-apirules.yml') as flask_client: - # Test happy path - response = flask_client.get(f'{flask_prefix}/conformance') - assert response.status_code == 200 - assert response.headers['X-API-Version'] == __version__ - assert response.request.url == \ - flask_client.application.url_for('pygeoapi.conformance') - response = flask_client.get(f'{flask_prefix}/static/img/pygeoapi.png') - assert response.status_code == 200 - # Test that static resources also work without URL prefix - response = flask_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test strict slashes - response = flask_client.get(f'{flask_prefix}/conformance/') - assert response.status_code == 404 - # For the landing page ONLY, trailing slashes are actually preferred. - # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa - # Omitting the trailing slash should lead to a redirect. - response = flask_client.get(f'{flask_prefix}/') - assert response.status_code == 200 - response = flask_client.get(flask_prefix) - assert response.status_code in (307, 308) - - # Test links on landing page for correct URLs - response = flask_client.get(flask_prefix, follow_redirects=True) - assert response.status_code == 200 - assert response.is_json - links = response.json['links'] - assert all( - href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa - ) - - # Test Starlette - starlette_prefix = rules.get_url_prefix('starlette') - with mock_starlette('pygeoapi-test-config-apirules.yml') as starlette_client: # noqa - # Test happy path - response = starlette_client.get(f'{starlette_prefix}/conformance') - assert response.status_code == 200 - assert response.headers['X-API-Version'] == __version__ - response = starlette_client.get(f'{starlette_prefix}/static/img/pygeoapi.png') # noqa - assert response.status_code == 200 - # Test that static resources also work without URL prefix - response = starlette_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test strict slashes - response = starlette_client.get(f'{starlette_prefix}/conformance/') - assert response.status_code == 404 - # For the landing page ONLY, trailing slashes are actually preferred. - # See https://docs.opengeospatial.org/is/17-069r4/17-069r4.html#_api_landing_page # noqa - # Omitting the trailing slash should lead to a redirect. - response = starlette_client.get(f'{starlette_prefix}/') - assert response.status_code == 200 - response = starlette_client.get(starlette_prefix) - assert response.status_code in (307, 308) - - # Test links on landing page for correct URLs - response = starlette_client.get(starlette_prefix, follow_redirects=True) # noqa - assert response.status_code == 200 - links = response.json()['links'] - assert all( - href.startswith(base_url) for href in (rel['href'] for rel in links) # noqa - ) - - -def test_apirules_inactive(config, api_): - assert api_.config == config - rules = get_api_rules(config) - - # Test Flask - flask_prefix = rules.get_url_prefix('flask') - assert flask_prefix == '' - with mock_flask('pygeoapi-test-config.yml') as flask_client: - response = flask_client.get('') - assert response.status_code == 200 - response = flask_client.get('/conformance') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - assert response.request.url == \ - flask_client.application.url_for('pygeoapi.conformance') - response = flask_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test trailing slashes - response = flask_client.get('/') - assert response.status_code == 200 - response = flask_client.get('/conformance/') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - - # Test Starlette - starlette_prefix = rules.get_url_prefix('starlette') - assert starlette_prefix == '' - with mock_starlette('pygeoapi-test-config.yml') as starlette_client: - response = starlette_client.get('') - assert response.status_code == 200 - response = starlette_client.get('/conformance') - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - assert str(response.url) == f"{starlette_client.base_url}/conformance" - response = starlette_client.get('/static/img/pygeoapi.png') - assert response.status_code == 200 - - # Test trailing slashes - response = starlette_client.get('/') - assert response.status_code == 200 - response = starlette_client.get('/conformance/', follow_redirects=True) - assert response.status_code == 200 - assert 'X-API-Version' not in response.headers - - -def test_api(config, api_, openapi): - assert api_.config == config - assert isinstance(api_.config, dict) - - req = mock_request(HTTP_ACCEPT='application/json') - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == 'application/vnd.oai.openapi+json;version=3.0' # noqa - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - root = json.loads(response) - assert isinstance(root, dict) - - a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - req = mock_request(HTTP_ACCEPT=a) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ - FORMAT_TYPES[F_HTML] - - assert 'Swagger UI' in response - - a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - req = mock_request({'ui': 'redoc'}, HTTP_ACCEPT=a) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] == \ - FORMAT_TYPES[F_HTML] - - assert 'ReDoc' in response - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.openapi_(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - assert api_.get_collections_url() == 'http://localhost:5000/collections' - - -def test_api_exception(config, api_): - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - # When a language is set, the exception should still be English - req = mock_request({'f': 'foo', 'lang': 'fr'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Language'] == 'en-US' - assert code == HTTPStatus.BAD_REQUEST - - -def test_gzip(config, api_): - # Requests for each response type and gzip encoding - req_gzip_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML], - HTTP_ACCEPT_ENCODING=F_GZIP) - req_gzip_gzip = mock_request(HTTP_ACCEPT='application/gzip', - HTTP_ACCEPT_ENCODING=F_GZIP) - - # Responses from server config without gzip compression - rsp_headers, _, rsp_json = api_.landing_page(req_gzip_json) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - rsp_headers, _, rsp_jsonld = api_.landing_page(req_gzip_jsonld) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - rsp_headers, _, rsp_html = api_.landing_page(req_gzip_html) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - rsp_headers, _, _ = api_.landing_page(req_gzip_gzip) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - - # Add gzip to server and use utf-16 encoding - config['server']['gzip'] = True - enc_16 = 'utf-16' - config['server']['encoding'] = enc_16 - api_ = API(config, openapi) - - # Responses from server with gzip compression - rsp_json_headers, _, rsp_gzip_json = api_.landing_page(req_gzip_json) - rsp_jsonld_headers, _, rsp_gzip_jsonld = api_.landing_page(req_gzip_jsonld) - rsp_html_headers, _, rsp_gzip_html = api_.landing_page(req_gzip_html) - rsp_gzip_headers, _, rsp_gzip_gzip = api_.landing_page(req_gzip_gzip) - - # Validate compressed json response - assert rsp_json_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_JSON]}; charset={enc_16}' - assert rsp_json_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_json = gzip.decompress(rsp_gzip_json).decode(enc_16) - assert isinstance(parsed_gzip_json, str) - parsed_gzip_json = json.loads(parsed_gzip_json) - assert isinstance(parsed_gzip_json, dict) - assert parsed_gzip_json == json.loads(rsp_json) - - # Validate compressed jsonld response - assert rsp_jsonld_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_JSONLD]}; charset={enc_16}' - assert rsp_jsonld_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_jsonld = gzip.decompress(rsp_gzip_jsonld).decode(enc_16) - assert isinstance(parsed_gzip_jsonld, str) - parsed_gzip_jsonld = json.loads(parsed_gzip_jsonld) - assert isinstance(parsed_gzip_jsonld, dict) - assert parsed_gzip_jsonld == json.loads(rsp_jsonld) - - # Validate compressed html response - assert rsp_html_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_HTML]}; charset={enc_16}' - assert rsp_html_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_html = gzip.decompress(rsp_gzip_html).decode(enc_16) - assert isinstance(parsed_gzip_html, str) - assert parsed_gzip_html == rsp_html - - # Validate compressed gzip response - assert rsp_gzip_headers['Content-Type'] == \ - f'{FORMAT_TYPES[F_GZIP]}; charset={enc_16}' - assert rsp_gzip_headers['Content-Encoding'] == F_GZIP - - parsed_gzip_gzip = gzip.decompress(rsp_gzip_gzip).decode(enc_16) - assert isinstance(parsed_gzip_gzip, str) - parsed_gzip_gzip = json.loads(parsed_gzip_gzip) - assert isinstance(parsed_gzip_gzip, dict) - - # Requests without content encoding header - req_json = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSON]) - req_jsonld = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_JSONLD]) - req_html = mock_request(HTTP_ACCEPT=FORMAT_TYPES[F_HTML]) - - # Responses without content encoding - _, _, rsp_json_ = api_.landing_page(req_json) - _, _, rsp_jsonld_ = api_.landing_page(req_jsonld) - _, _, rsp_html_ = api_.landing_page(req_html) - - # Confirm each request is the same when decompressed - assert rsp_json_ == rsp_json == \ - gzip.decompress(rsp_gzip_json).decode(enc_16) - - assert rsp_jsonld_ == rsp_jsonld == \ - gzip.decompress(rsp_gzip_jsonld).decode(enc_16) - - assert rsp_html_ == rsp_html == \ - gzip.decompress(rsp_gzip_html).decode(enc_16) - - -def test_gzip_csv(config, api_): - req_csv = mock_request({'f': 'csv'}) - rsp_csv_headers, _, rsp_csv = api_.get_collection_items(req_csv, 'obs') - assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' - rsp_csv = rsp_csv.decode('utf-8') - - req_csv = mock_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) - rsp_csv_headers, _, rsp_csv_gzip = api_.get_collection_items(req_csv, 'obs') # noqa - assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' - rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') - assert rsp_csv == rsp_csv_ - - # Use utf-16 encoding - config['server']['encoding'] = 'utf-16' - api_ = API(config, openapi) - - req_csv = mock_request({'f': 'csv'}, HTTP_ACCEPT_ENCODING=F_GZIP) - rsp_csv_headers, _, rsp_csv_gzip = api_.get_collection_items(req_csv, 'obs') # noqa - assert rsp_csv_headers['Content-Type'] == 'text/csv; charset=utf-8' - rsp_csv_ = gzip.decompress(rsp_csv_gzip).decode('utf-8') - assert rsp_csv == rsp_csv_ - - -def test_root(config, api_): - req = mock_request() - rsp_headers, code, response = api_.landing_page(req) - root = json.loads(response) - - assert rsp_headers['Content-Type'] == 'application/json' == \ - FORMAT_TYPES[F_JSON] - assert rsp_headers['X-Powered-By'].startswith('pygeoapi') - assert rsp_headers['Content-Language'] == 'en-US' - - assert isinstance(root, dict) - assert 'links' in root - assert root['links'][0]['rel'] == 'self' - assert root['links'][0]['type'] == FORMAT_TYPES[F_JSON] - assert root['links'][0]['href'].endswith('?f=json') - assert any(link['href'].endswith('f=jsonld') and link['rel'] == 'alternate' - for link in root['links']) - assert any(link['href'].endswith('f=html') and link['rel'] == 'alternate' - for link in root['links']) - assert len(root['links']) == 11 - assert 'title' in root - assert root['title'] == 'pygeoapi default instance' - assert 'description' in root - assert root['description'] == 'pygeoapi provides an API to geospatial data' - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.landing_page(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_root_structured_data(config, api_): - req = mock_request({"f": "jsonld"}) - rsp_headers, code, response = api_.landing_page(req) - root = json.loads(response) - - assert rsp_headers['Content-Type'] == 'application/ld+json' == \ - FORMAT_TYPES[F_JSONLD] - assert rsp_headers['Content-Language'] == 'en-US' - assert rsp_headers['X-Powered-By'].startswith('pygeoapi') - - assert isinstance(root, dict) - assert 'description' in root - assert root['description'] == 'pygeoapi provides an API to geospatial data' - - assert '@context' in root - assert root['@context'] == 'https://schema.org/docs/jsonldcontext.jsonld' - expanded = jsonld.expand(root)[0] - assert '@type' in expanded - assert 'http://schema.org/DataCatalog' in expanded['@type'] - assert 'http://schema.org/description' in expanded - assert root['description'] == expanded['http://schema.org/description'][0][ - '@value'] - assert 'http://schema.org/keywords' in expanded - assert len(expanded['http://schema.org/keywords']) == 3 - assert '@value' in expanded['http://schema.org/keywords'][0].keys() - assert 'http://schema.org/provider' in expanded - assert expanded['http://schema.org/provider'][0]['@type'][ - 0] == 'http://schema.org/Organization' - assert expanded['http://schema.org/name'][0]['@value'] == root['name'] - - -def test_conformance(config, api_): - req = mock_request() - rsp_headers, code, response = api_.conformance(req) - root = json.loads(response) - - assert isinstance(root, dict) - assert 'conformsTo' in root - assert len(root['conformsTo']) == 36 - assert 'http://www.opengis.net/spec/ogcapi-features-2/1.0/conf/crs' \ - in root['conformsTo'] - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.conformance(req) - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.conformance(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_tilematrixsets(config, api_): - req = mock_request() - rsp_headers, code, response = api_.tilematrixsets(req) - root = json.loads(response) - - assert isinstance(root, dict) - assert 'tileMatrixSets' in root - assert len(root['tileMatrixSets']) == 2 - assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WorldCRS84Quad' \ - in root['tileMatrixSets'][0]['uri'] - assert 'http://www.opengis.net/def/tilematrixset/OGC/1.0/WebMercatorQuad' \ - in root['tileMatrixSets'][1]['uri'] - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.tilematrixsets(req) - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.tilematrixsets(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_tilematrixset(config, api_): - req = mock_request() - - enums = [e.value for e in TileMatrixSetEnum] - enum = None - - for e in enums: - enum = e.tileMatrixSet - rsp_headers, code, response = api_.tilematrixset(req, enum) - root = json.loads(response) - - assert isinstance(root, dict) - assert 'id' in root - assert root['id'] == enum - assert 'tileMatrices' in root - assert len(root['tileMatrices']) == 30 - - rsp_headers, code, response = api_.tilematrixset(req, 'foo') - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.tilematrixset(req, enum) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_describe_collections(config, api_): - req = mock_request({"f": "foo"}) - rsp_headers, code, response = api_.describe_collections(req) - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({"f": "html"}) - rsp_headers, code, response = api_.describe_collections(req) - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - - req = mock_request() - rsp_headers, code, response = api_.describe_collections(req) - collections = json.loads(response) - - assert len(collections) == 2 - assert len(collections['collections']) == 9 - assert len(collections['links']) == 3 - - rsp_headers, code, response = api_.describe_collections(req, 'foo') - collection = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert rsp_headers['Content-Language'] == 'en-US' - assert collection['id'] == 'obs' - assert collection['title'] == 'Observations' - assert collection['description'] == 'My cool observations' - assert len(collection['links']) == 14 - assert collection['extent'] == { - 'spatial': { - 'bbox': [[-180, -90, 180, 90]], - 'crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' - }, - 'temporal': { - 'interval': [ - ['2000-10-30T18:24:39+00:00', '2007-10-30T08:57:29+00:00'] - ], - 'trs': 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian' - } - } - - # OAPIF Part 2 CRS 6.2.1 A, B, configured CRS + defaults - assert collection['crs'] is not None - crs_set = [ - 'http://www.opengis.net/def/crs/EPSG/0/28992', - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', - 'http://www.opengis.net/def/crs/EPSG/0/4326', - ] - for crs in crs_set: - assert crs in collection['crs'] - assert collection['storageCRS'] is not None - assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa - assert 'storageCrsCoordinateEpoch' not in collection - - # French language request - req = mock_request({'lang': 'fr'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert rsp_headers['Content-Language'] == 'fr-CA' - assert collection['title'] == 'Observations' - assert collection['description'] == 'Mes belles observations' - - # Check HTML request in an unsupported language - req = mock_request({'f': 'html', 'lang': 'de'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'en-US' - - req = mock_request() - rsp_headers, code, response = api_.describe_collections(req, - 'gdps-temperature') - collection = json.loads(response) - - assert collection['id'] == 'gdps-temperature' - assert len(collection['links']) == 10 - assert collection['extent']['spatial']['grid'][0]['cellsCount'] == 2400 - assert collection['extent']['spatial']['grid'][0]['resolution'] == 0.15000000000000002 # noqa - assert collection['extent']['spatial']['grid'][1]['cellsCount'] == 1201 - assert collection['extent']['spatial']['grid'][1]['resolution'] == 0.15 - - # hiearchical collections - rsp_headers, code, response = api_.describe_collections( - req, 'naturalearth/lakes') - collection = json.loads(response) - assert collection['id'] == 'naturalearth/lakes' - - # OAPIF Part 2 CRS 6.2.1 B, defaults when not configured - assert collection['crs'] is not None - default_crs_list = [ - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84', - 'http://www.opengis.net/def/crs/OGC/1.3/CRS84h', - ] - contains_default = False - for crs in default_crs_list: - if crs in default_crs_list: - contains_default = True - assert contains_default - assert collection['storageCRS'] is not None - assert collection['storageCRS'] == 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' # noqa - assert collection['storageCrsCoordinateEpoch'] == 2017.23 - - -def test_describe_collections_hidden_resources( - config_hidden_resources, api_hidden_resources): - req = mock_request({}) - rsp_headers, code, response = api_hidden_resources.describe_collections(req) # noqa - assert code == HTTPStatus.OK - - assert len(config_hidden_resources['resources']) == 3 - - collections = json.loads(response) - assert len(collections['collections']) == 1 - - -def test_get_collection_schema(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_schema(req, - 'notfound') - assert code == HTTPStatus.NOT_FOUND - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_schema(req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_schema(req, 'obs') - assert rsp_headers['Content-Type'] == 'application/schema+json' - schema = json.loads(response) - - assert 'properties' in schema - assert len(schema['properties']) == 5 - - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_schema( - req, 'gdps-temperature') - assert rsp_headers['Content-Type'] == 'application/schema+json' - schema = json.loads(response) - - assert 'properties' in schema - assert len(schema['properties']) == 1 - assert schema['properties']['1']['type'] == 'number' - - -def test_get_collection_queryables(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_queryables(req, - 'notfound') - assert code == HTTPStatus.NOT_FOUND - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_queryables(req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_queryables(req, 'obs') - assert rsp_headers['Content-Type'] == 'application/schema+json' - queryables = json.loads(response) - - assert 'properties' in queryables - assert len(queryables['properties']) == 5 - - # test with provider filtered properties - api_.config['resources']['obs']['providers'][0]['properties'] = ['stn_id'] - - rsp_headers, code, response = api_.get_collection_queryables(req, 'obs') - queryables = json.loads(response) - - assert 'properties' in queryables - assert len(queryables['properties']) == 2 - assert 'geometry' in queryables['properties'] - assert queryables['properties']['geometry']['$ref'] == 'https://geojson.org/schema/Geometry.json' # noqa - - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_describe_collections_json_ld(config, api_): - req = mock_request({'f': 'jsonld'}) - rsp_headers, code, response = api_.describe_collections(req, 'obs') - collection = json.loads(response) - - assert '@context' in collection - expanded = jsonld.expand(collection)[0] - # Metadata is about a schema:DataCollection that contains a schema:Dataset - assert not expanded['@id'].endswith('obs') - assert 'http://schema.org/dataset' in expanded - assert len(expanded['http://schema.org/dataset']) == 1 - dataset = expanded['http://schema.org/dataset'][0] - assert dataset['@type'][0] == 'http://schema.org/Dataset' - assert len(dataset['http://schema.org/distribution']) == 14 - assert all(dist['@type'][0] == 'http://schema.org/DataDownload' - for dist in dataset['http://schema.org/distribution']) - - assert 'http://schema.org/Organization' in expanded[ - 'http://schema.org/provider'][0]['@type'] - - assert 'http://schema.org/Place' in dataset[ - 'http://schema.org/spatial'][0]['@type'] - assert 'http://schema.org/GeoShape' in dataset[ - 'http://schema.org/spatial'][0]['http://schema.org/geo'][0]['@type'] - assert dataset['http://schema.org/spatial'][0]['http://schema.org/geo'][ - 0]['http://schema.org/box'][0]['@value'] == '-180,-90 180,90' - - assert 'http://schema.org/temporalCoverage' in dataset - assert dataset['http://schema.org/temporalCoverage'][0][ - '@value'] == '2000-10-30T18:24:39+00:00/2007-10-30T08:57:29+00:00' - - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - -def test_get_collection_items(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_items(req, 'foo') - features = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'bbox': '1,2,3'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'bbox': '1,2,3,4c'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'bbox': '1,2,3,4', 'bbox-crs': 'bad_value'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'bbox-crs': 'bad_value'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - # bbox-crs must be in configured values for Collection - req = mock_request({'bbox': '1,2,3,4', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4258'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - # bbox-crs must be in configured values for Collection (CSV will ignore) - req = mock_request({'bbox': '52,4,53,5', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - # bbox-crs can be a default even if not configured - req = mock_request({'bbox': '4,52,5,53', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - # bbox-crs can be a default even if not configured - req = mock_request({'bbox': '4,52,5,53'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'f': 'html', 'lang': 'fr'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'fr-CA' - - req = mock_request() - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - # No language requested: should be set to default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - assert len(features['features']) == 5 - - req = mock_request({'resulttype': 'hits'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 0 - - # Invalid limit - req = mock_request({'limit': 0}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'stn_id': '35'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 2 - assert features['numberMatched'] == 2 - - req = mock_request({'stn_id': '35', 'value': '93.9'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 1 - assert features['numberMatched'] == 1 - - req = mock_request({'limit': 2}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 2 - assert features['features'][1]['properties']['stn_id'] == 35 - - links = features['links'] - assert len(links) == 4 - assert '/collections/obs/items?f=json' in links[0]['href'] - assert links[0]['rel'] == 'self' - assert '/collections/obs/items?f=jsonld' in links[1]['href'] - assert links[1]['rel'] == 'alternate' - assert '/collections/obs/items?f=html' in links[2]['href'] - assert links[2]['rel'] == 'alternate' - assert '/collections/obs' in links[3]['href'] - assert links[3]['rel'] == 'collection' - - # Invalid offset - req = mock_request({'offset': -1}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'offset': 2}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 3 - assert features['features'][1]['properties']['stn_id'] == 2147 - - links = features['links'] - assert len(links) == 5 - assert '/collections/obs/items?f=json' in links[0]['href'] - assert links[0]['rel'] == 'self' - assert '/collections/obs/items?f=jsonld' in links[1]['href'] - assert links[1]['rel'] == 'alternate' - assert '/collections/obs/items?f=html' in links[2]['href'] - assert links[2]['rel'] == 'alternate' - assert '/collections/obs/items?offset=0' in links[3]['href'] - assert links[3]['rel'] == 'prev' - assert '/collections/obs' in links[4]['href'] - assert links[4]['rel'] == 'collection' - - req = mock_request({ - 'offset': 1, - 'limit': 1, - 'bbox': '-180,90,180,90' - }) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - - assert len(features['features']) == 1 - - links = features['links'] - assert len(links) == 5 - assert '/collections/obs/items?f=json&limit=1&bbox=-180,90,180,90' in \ - links[0]['href'] - assert links[0]['rel'] == 'self' - assert '/collections/obs/items?f=jsonld&limit=1&bbox=-180,90,180,90' in \ - links[1]['href'] - assert links[1]['rel'] == 'alternate' - assert '/collections/obs/items?f=html&limit=1&bbox=-180,90,180,90' in \ - links[2]['href'] - assert links[2]['rel'] == 'alternate' - assert '/collections/obs/items?offset=0&limit=1&bbox=-180,90,180,90' \ - in links[3]['href'] - assert links[3]['rel'] == 'prev' - assert '/collections/obs' in links[4]['href'] - assert links[4]['rel'] == 'collection' - - req = mock_request({ - 'sortby': 'bad-property', - 'stn_id': '35' - }) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'sortby': 'stn_id'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - assert code == HTTPStatus.OK - - req = mock_request({'sortby': '+stn_id'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - assert code == HTTPStatus.OK - - req = mock_request({'sortby': '-stn_id'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - features = json.loads(response) - assert code == HTTPStatus.OK - - req = mock_request({'f': 'csv'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert rsp_headers['Content-Type'] == 'text/csv; charset=utf-8' - - req = mock_request({'datetime': '2003'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'datetime': '1999'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'datetime': '2010-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'datetime': '2001-11-11/2003-12-18'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'datetime': '../2003-12-18'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'datetime': '2001-11-11/..'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'datetime': '1999/2005-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'datetime': '1999/2000-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - api_.config['resources']['obs']['extents'].pop('temporal') - - req = mock_request({'datetime': '2002/2014-04-22'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.OK - - req = mock_request({'scalerank': 1}) - rsp_headers, code, response = api_.get_collection_items( - req, 'naturalearth/lakes') - features = json.loads(response) - - assert len(features['features']) == 10 - assert features['numberMatched'] == 11 - assert features['numberReturned'] == 10 - - req = mock_request({'datetime': '2005-04-22'}) - rsp_headers, code, response = api_.get_collection_items( - req, 'naturalearth/lakes') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'skipGeometry': 'true'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert json.loads(response)['features'][0]['geometry'] is None - - req = mock_request({'properties': 'foo,bar'}) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - -def test_get_collection_items_crs(config, api_): - - # Invalid CRS query parameter - req = mock_request({'crs': '4326'}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') - - assert code == HTTPStatus.BAD_REQUEST - - # Unsupported CRS - req = mock_request({'crs': 'http://www.opengis.net/def/crs/EPSG/0/32633'}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') - - assert code == HTTPStatus.BAD_REQUEST - - # Supported CRSs - default_crs = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' - storage_crs = 'http://www.opengis.net/def/crs/EPSG/0/25833' - crs_4258 = 'http://www.opengis.net/def/crs/EPSG/0/4258' - supported_crs_list = [default_crs, storage_crs, crs_4258] - - for crs in supported_crs_list: - req = mock_request({'crs': crs}) - rsp_headers, code, response = api_.get_collection_items( - req, 'norway_pop', - ) - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{crs}>' - - # With CRS query parameter, using storageCRS - req = mock_request({'crs': storage_crs}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' - - features_25833 = json.loads(response) - - # With CRS query parameter resulting in coordinates transformation - req = mock_request({'crs': crs_4258}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{crs_4258}>' - - features_4258 = json.loads(response) - transform_func = pyproj.Transformer.from_crs( - pyproj.CRS.from_epsg(25833), - pyproj.CRS.from_epsg(4258), - always_xy=False, - ).transform - for feat_orig in features_25833['features']: - id_ = feat_orig['id'] - x, y, *_ = feat_orig['geometry']['coordinates'] - loc_transf = Point(transform_func(x, y)) - for feat_out in features_4258['features']: - if id_ == feat_out['id']: - loc_out = Point(feat_out['geometry']['coordinates'][:2]) - - assert loc_out.equals_exact(loc_transf, 1e-5) - break - - # Without CRS query parameter: assume Transform to default WGS84 lon,lat - req = mock_request({}) - rsp_headers, code, response = api_.get_collection_items(req, 'norway_pop') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Crs'] == f'<{default_crs}>' - - features_wgs84 = json.loads(response) - - # With CRS query parameter resulting in coordinates transformation - transform_func = pyproj.Transformer.from_crs( - pyproj.CRS.from_epsg(4258), - get_crs_from_uri(default_crs), - always_xy=False, - ).transform - for feat_orig in features_4258['features']: - id_ = feat_orig['id'] - x, y, *_ = feat_orig['geometry']['coordinates'] - loc_transf = Point(transform_func(x, y)) - for feat_out in features_wgs84['features']: - if id_ == feat_out['id']: - loc_out = Point(feat_out['geometry']['coordinates'][:2]) - - assert loc_out.equals_exact(loc_transf, 1e-5) - break - - -def test_manage_collection_item_read_only_options_req(config, api_): - """Test OPTIONS request on a read-only items endpoint""" - req = mock_request() - _, code, _ = api_.manage_collection_item(req, 'options', 'foo') - assert code == HTTPStatus.NOT_FOUND - - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item(req, 'options', 'obs') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET' - - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item( - req, 'options', 'obs', 'ressource_id') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET' - - -def test_manage_collection_item_editable_options_req(config): - """Test OPTIONS request on a editable items endpoint""" - config = copy.deepcopy(config) - config['resources']['obs']['providers'][0]['editable'] = True - api_ = API(config, openapi) - - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item(req, 'options', 'obs') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET, POST' - - req = mock_request() - rsp_headers, code, _ = api_.manage_collection_item( - req, 'options', 'obs', 'ressource_id') - assert code == HTTPStatus.OK - assert rsp_headers['Allow'] == 'HEAD, GET, PUT, DELETE' - - -def test_describe_collections_enclosures(config_enclosure, enclosure_api): - original_enclosures = { - lnk['title']: lnk - for lnk in config_enclosure['resources']['objects']['links'] - if lnk['rel'] == 'enclosure' - } - - req = mock_request() - _, _, response = enclosure_api.describe_collections(req, 'objects') - features = json.loads(response) - modified_enclosures = { - lnk['title']: lnk for lnk in features['links'] - if lnk['rel'] == 'enclosure' - } - - # If type and length is set, do not verify/update link - assert original_enclosures['download link 1'] == \ - modified_enclosures['download link 1'] - # If length is missing, modify link type and length - assert original_enclosures['download link 2']['type'] == \ - modified_enclosures['download link 2']['type'] - assert modified_enclosures['download link 2']['type'] == \ - modified_enclosures['download link 3']['type'] - assert 'length' not in original_enclosures['download link 2'] - assert modified_enclosures['download link 2']['length'] > 0 - assert modified_enclosures['download link 2']['length'] == \ - modified_enclosures['download link 3']['length'] - assert original_enclosures['download link 3']['type'] != \ - modified_enclosures['download link 3']['type'] - - -def test_get_collection_items_json_ld(config, api_): - req = mock_request({ - 'f': 'jsonld', - 'limit': 2 - }) - rsp_headers, code, response = api_.get_collection_items(req, 'obs') - - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - # No language requested: return default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - collection = json.loads(response) - - assert '@context' in collection - assert all((f in collection['@context'][0] for - f in ('schema', 'type', 'features', 'FeatureCollection'))) - assert len(collection['@context']) > 1 - assert collection['@context'][1]['schema'] == 'https://schema.org/' - expanded = jsonld.expand(collection)[0] - featuresUri = 'https://schema.org/itemListElement' - assert len(expanded[featuresUri]) == 2 - - -def test_get_collection_item(config, api_): - req = mock_request({'f': 'foo'}) - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.get_collection_item( - req, 'gdps-temperature', '371') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request() - rsp_headers, code, response = api_.get_collection_item(req, 'foo', '371') - - assert code == HTTPStatus.NOT_FOUND - - rsp_headers, code, response = api_.get_collection_item( - req, 'obs', 'notfound') - - assert code == HTTPStatus.NOT_FOUND - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') - - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - assert rsp_headers['Content-Language'] == 'en-US' - - req = mock_request() - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') - feature = json.loads(response) - - assert feature['properties']['stn_id'] == 35 - assert 'prev' not in feature['links'] - assert 'next' not in feature['links'] - - -def test_get_collection_item_json_ld(config, api_): - req = mock_request({'f': 'jsonld'}) - rsp_headers, _, response = api_.get_collection_item(req, 'objects', '3') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - assert rsp_headers['Content-Language'] == 'en-US' - feature = json.loads(response) - assert '@context' in feature - assert all((f in feature['@context'][0] for - f in ('schema', 'type', 'gsp'))) - assert len(feature['@context']) == 1 - assert 'schema' in feature['@context'][0] - assert feature['@context'][0]['schema'] == 'https://schema.org/' - assert feature['id'] == 3 - expanded = jsonld.expand(feature)[0] - - assert expanded['@id'].startswith('http://') - assert expanded['@id'].endswith('/collections/objects/items/3') - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'POINT (-85 33)' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/latitude'][0][ - '@value'] == 33 - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/longitude'][0][ - '@value'] == -85 - - _, _, response = api_.get_collection_item(req, 'objects', '2') - feature = json.loads(response) - assert feature['geometry']['type'] == 'MultiPoint' - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'MULTIPOINT (10 40, 40 30, 20 20, 30 10)' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/polygon'][0][ - '@value'] == "10.0,40.0 40.0,30.0 20.0,20.0 30.0,10.0 10.0,40.0" - - _, _, response = api_.get_collection_item(req, 'objects', '1') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'LINESTRING (30 10, 10 30, 40 40)' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/line'][0][ - '@value'] == '30.0,10.0 10.0,30.0 40.0,40.0' - - _, _, response = api_.get_collection_item(req, 'objects', '4') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'MULTILINESTRING ((10 10, 20 20, 10 40), ' \ - '(40 40, 30 30, 40 20, 30 10))' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/line'][0][ - '@value'] == '10.0,10.0 20.0,20.0 10.0,40.0 40.0,40.0 ' \ - '30.0,30.0 40.0,20.0 30.0,10.0' - - _, _, response = api_.get_collection_item(req, 'objects', '5') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/polygon'][0][ - '@value'] == '30.0,10.0 40.0,40.0 20.0,40.0 10.0,20.0 30.0,10.0' - - _, _, response = api_.get_collection_item(req, 'objects', '7') - feature = json.loads(response) - expanded = jsonld.expand(feature)[0] - assert expanded['http://www.opengis.net/ont/geosparql#hasGeometry'][0][ - 'http://www.opengis.net/ont/geosparql#asWKT'][0][ - '@value'] == 'MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), '\ - '((15 5, 40 10, 10 20, 5 10, 15 5)))' - assert expanded['https://schema.org/geo'][0][ - 'https://schema.org/polygon'][0][ - '@value'] == '15.0,5.0 5.0,10.0 10.0,40.0 '\ - '45.0,40.0 40.0,10.0 15.0,5.0' - - req = mock_request({'f': 'jsonld', 'lang': 'fr'}) - rsp_headers, code, response = api_.get_collection_item(req, 'obs', '371') - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSONLD] - assert rsp_headers['Content-Language'] == 'fr-CA' - - -def test_get_collection_coverage(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_coverage( - req, 'obs') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'properties': '12'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'subset': 'bad_axis(10:20)'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'blah'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.BAD_REQUEST - assert rsp_headers['Content-Type'] == 'text/html' - - req = mock_request(HTTP_ACCEPT='text/html') - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - # NOTE: This test used to assert the code to be 200 OK, - # but it requested HTML, which is not available, - # so it should be 400 Bad Request - assert code == HTTPStatus.BAD_REQUEST - assert rsp_headers['Content-Type'] == 'text/html' - - req = mock_request({'subset': 'Lat(5:10),Long(5:10)'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.OK - content = json.loads(response) - - assert content['domain']['axes']['x']['num'] == 35 - assert content['domain']['axes']['y']['num'] == 35 - assert 'TMP' in content['parameters'] - assert 'TMP' in content['ranges'] - assert content['ranges']['TMP']['axisNames'] == ['y', 'x'] - - req = mock_request({'bbox': '-79,45,-75,49'}) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.OK - content = json.loads(response) - - assert content['domain']['axes']['x']['start'] == -79.0 - assert content['domain']['axes']['x']['stop'] == -75.0 - assert content['domain']['axes']['y']['start'] == 49.0 - assert content['domain']['axes']['y']['stop'] == 45.0 - - req = mock_request({ - 'subset': 'Lat(5:10),Long(5:10)', - 'f': 'GRIB' - }) - rsp_headers, code, response = api_.get_collection_coverage( - req, 'gdps-temperature') - - assert code == HTTPStatus.OK - assert isinstance(response, bytes) - - req = mock_request(HTTP_ACCEPT='application/x-netcdf') - rsp_headers, code, response = api_.get_collection_coverage( - req, 'cmip5') - - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == 'application/x-netcdf' - - # req = mock_request({ - # 'subset': 'time("2006-07-01T06:00:00":"2007-07-01T06:00:00")' - # }) - # rsp_headers, code, response = api_.get_collection_coverage(req, 'cmip5') - # - # assert code == HTTPStatus.OK - # assert isinstance(json.loads(response), dict) - - # req = mock_request({'subset': 'lat(1:2'}) - # rsp_headers, code, response = api_.get_collection_coverage(req, 'cmip5') - # - # assert code == HTTPStatus.BAD_REQUEST - # - # req = mock_request({'subset': 'lat(1:2)'}) - # rsp_headers, code, response = api_.get_collection_coverage(req, 'cmip5') - # - # assert code == HTTPStatus.NO_CONTENT - - -def test_get_collection_map(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_map(req, 'notfound') - assert code == HTTPStatus.NOT_FOUND - - req = mock_request() - rsp_headers, code, response = api_.get_collection_map( - req, 'mapserver_world_map') - assert code == HTTPStatus.OK - assert isinstance(response, bytes) - assert response[1:4] == b'PNG' - - -def test_get_collection_tiles(config, api_): - req = mock_request() - rsp_headers, code, response = api_.get_collection_tiles(req, 'obs') - assert code == HTTPStatus.BAD_REQUEST - - rsp_headers, code, response = api_.get_collection_tiles( - req, 'naturalearth/lakes') - assert code == HTTPStatus.OK - - # Language settings should be ignored (return system default) - req = mock_request({'lang': 'fr'}) - rsp_headers, code, response = api_.get_collection_tiles( - req, 'naturalearth/lakes') - assert rsp_headers['Content-Language'] == 'en-US' - content = json.loads(response) - assert len(content['links']) > 0 - assert len(content['tilesets']) > 0 - - -def test_describe_processes(config, api_): - req = mock_request({'limit': 1}) - # Test for description of single processes - rsp_headers, code, response = api_.describe_processes(req) - data = json.loads(response) - assert code == HTTPStatus.OK - assert len(data['processes']) == 1 - assert len(data['links']) == 3 - - req = mock_request() - - # Test for undefined process - rsp_headers, code, response = api_.describe_processes(req, 'foo') - data = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert data['code'] == 'NoSuchProcess' - - # Test for description of all processes - rsp_headers, code, response = api_.describe_processes(req) - data = json.loads(response) - assert code == HTTPStatus.OK - assert len(data['processes']) == 2 - assert len(data['links']) == 3 - - # Test for particular, defined process - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - process = json.loads(response) - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert process['id'] == 'hello-world' - assert process['version'] == '0.2.0' - assert process['title'] == 'Hello World' - assert len(process['keywords']) == 3 - assert len(process['links']) == 6 - assert len(process['inputs']) == 2 - assert len(process['outputs']) == 1 - assert len(process['outputTransmission']) == 1 - assert len(process['jobControlOptions']) == 2 - assert 'sync-execute' in process['jobControlOptions'] - assert 'async-execute' in process['jobControlOptions'] - - # Check HTML response when requested in headers - req = mock_request(HTTP_ACCEPT='text/html') - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: return default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - # Check JSON response when requested in headers - req = mock_request(HTTP_ACCEPT='application/json') - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'en-US' - - # Check HTML response when requested with query parameter - req = mock_request({'f': 'html'}) - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_HTML] - # No language requested: return default from YAML - assert rsp_headers['Content-Language'] == 'en-US' - - # Check JSON response when requested with query parameter - req = mock_request({'f': 'json'}) - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'en-US' - - # Check JSON response when requested with French language parameter - req = mock_request({'lang': 'fr'}) - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'fr-CA' - process = json.loads(response) - assert process['title'] == 'Bonjour le Monde' - - # Check JSON response when language requested in headers - req = mock_request(HTTP_ACCEPT_LANGUAGE='fr') - rsp_headers, code, response = api_.describe_processes(req, 'hello-world') - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - assert rsp_headers['Content-Language'] == 'fr-CA' - - # Test for undefined process - req = mock_request() - rsp_headers, code, response = api_.describe_processes(req, 'goodbye-world') - data = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert data['code'] == 'NoSuchProcess' - assert rsp_headers['Content-Type'] == FORMAT_TYPES[F_JSON] - - # Test describe doesn't crash if example is missing - req = mock_request() - processor = api_.manager.get_processor("hello-world") - example = processor.metadata.pop("example") - rsp_headers, code, response = api_.describe_processes(req) - processor.metadata['example'] = example - data = json.loads(response) - assert code == HTTPStatus.OK - assert len(data['processes']) == 2 - - -def test_execute_process(config, api_): - req_body_0 = { - 'inputs': { - 'name': 'Test' - } - } - req_body_1 = { - 'inputs': { - 'name': 'Test' - }, - 'response': 'document' - } - req_body_2 = { - 'inputs': { - 'name': 'Tést' - } - } - req_body_3 = { - 'inputs': { - 'name': 'Tést', - 'message': 'This is a test.' - } - } - req_body_4 = { - 'inputs': { - 'foo': 'Tést' - } - } - req_body_5 = { - 'inputs': {} - } - req_body_6 = { - 'inputs': { - 'name': None - } - } - req_body_7 = { - 'inputs': { - 'name': 'Test' - }, - 'subscriber': { - 'successUri': 'https://example.com/success', - 'inProgressUri': 'https://example.com/inProgress', - 'failedUri': 'https://example.com/failed', - } - } - - cleanup_jobs = set() - - # Test posting empty payload to existing process - req = mock_request(data='') - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - assert rsp_headers['Content-Language'] == 'en-US' - - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' not in rsp_headers - assert data['code'] == 'MissingParameterValue' - - req = mock_request(data=req_body_0) - rsp_headers, code, response = api_.execute_process(req, 'foo') - - data = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert 'Location' not in rsp_headers - assert data['code'] == 'NoSuchProcess' - - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - - assert len(data.keys()) == 2 - assert data['id'] == 'echo' - assert data['value'] == 'Hello Test!' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_1) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - - assert len(data.keys()) == 1 - assert data['outputs'][0]['id'] == 'echo' - assert data['outputs'][0]['value'] == 'Hello Test!' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_2) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - assert data['value'] == 'Hello Tést!' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_3) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - assert data['value'] == 'Hello Tést! This is a test.' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_4) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' in rsp_headers - assert data['code'] == 'InvalidParameterValue' - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_5) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' in rsp_headers - assert data['code'] == 'InvalidParameterValue' - assert data['description'] == 'Error updating job' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_6) - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.BAD_REQUEST - assert 'Location' in rsp_headers - assert data['code'] == 'InvalidParameterValue' - assert data['description'] == 'Error updating job' - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_0) - rsp_headers, code, response = api_.execute_process(req, 'goodbye-world') - - response = json.loads(response) - assert code == HTTPStatus.NOT_FOUND - assert 'Location' not in rsp_headers - assert response['code'] == 'NoSuchProcess' - - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - response = json.loads(response) - assert code == HTTPStatus.OK - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_1, HTTP_Prefer='respond-async') - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - - assert 'Location' in rsp_headers - response = json.loads(response) - assert isinstance(response, dict) - assert code == HTTPStatus.CREATED - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - req = mock_request(data=req_body_7) - with mock.patch( - 'pygeoapi.process.manager.base.requests.post' - ) as post_mocker: - rsp_headers, code, response = api_.execute_process(req, 'hello-world') - assert code == HTTPStatus.OK - post_mocker.assert_any_call( - req_body_7['subscriber']['inProgressUri'], json={} - ) - post_mocker.assert_any_call( - req_body_7['subscriber']['successUri'], - json={'id': 'echo', 'value': 'Hello Test!'} - ) - assert post_mocker.call_count == 2 - - cleanup_jobs.add(tuple(['hello-world', - rsp_headers['Location'].split('/')[-1]])) - - # Cleanup - time.sleep(2) # Allow time for any outstanding async jobs - for _, job_id in cleanup_jobs: - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) - assert code == HTTPStatus.OK - - -def _execute_a_job(api_): - req_body_sync = { - 'inputs': { - 'name': 'Sync Test' - } - } - - req = mock_request(data=req_body_sync) - rsp_headers, code, response = api_.execute_process( - req, 'hello-world') - - data = json.loads(response) - assert code == HTTPStatus.OK - assert 'Location' in rsp_headers - assert data['value'] == 'Hello Sync Test!' - - job_id = rsp_headers['Location'].split('/')[-1] - return job_id - - -def test_delete_job(api_): - rsp_headers, code, response = api_.delete_job( - mock_request(), 'does-not-exist') - - assert code == HTTPStatus.NOT_FOUND - req_body_async = { - 'inputs': { - 'name': 'Async Test Deletion' - } - } - job_id = _execute_a_job(api_) - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) - - assert code == HTTPStatus.OK - - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) - assert code == HTTPStatus.NOT_FOUND - - req = mock_request(data=req_body_async, HTTP_Prefer='respond-async') - rsp_headers, code, response = api_.execute_process( - req, 'hello-world') - - assert code == HTTPStatus.CREATED - assert 'Location' in rsp_headers - - time.sleep(2) # Allow time for async execution to complete - job_id = rsp_headers['Location'].split('/')[-1] - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) - assert code == HTTPStatus.OK - - rsp_headers, code, response = api_.delete_job(mock_request(), job_id) - assert code == HTTPStatus.NOT_FOUND - - -def test_get_job_result(api_): - rsp_headers, code, response = api_.get_job_result(mock_request(), - 'not-exist') - assert code == HTTPStatus.NOT_FOUND - - job_id = _execute_a_job(api_) - rsp_headers, code, response = api_.get_job_result(mock_request(), job_id) - # default response is html - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == 'text/html' - assert 'Hello Sync Test!' in response - - rsp_headers, code, response = api_.get_job_result( - mock_request({'f': 'json'}), job_id, - ) - assert code == HTTPStatus.OK - assert rsp_headers['Content-Type'] == 'application/json' - assert json.loads(response)['value'] == "Hello Sync Test!" - - -def test_get_collection_edr_query(config, api_): - # edr resource - req = mock_request() - rsp_headers, code, response = api_.describe_collections(req, 'icoads-sst') - collection = json.loads(response) - parameter_names = list(collection['parameter_names'].keys()) - parameter_names.sort() - assert len(parameter_names) == 4 - assert parameter_names == ['AIRT', 'SST', 'UWND', 'VWND'] - - # no coords parameter - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # bad query type - req = mock_request({'coords': 'POINT(11 11)'}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'corridor') - assert code == HTTPStatus.BAD_REQUEST - - # bad coords parameter - req = mock_request({'coords': 'gah'}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # bad parameter_names parameter - req = mock_request({ - 'coords': 'POINT(11 11)', 'parameter_names': 'bad' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # all parameters - req = mock_request({'coords': 'POINT(11 11)'}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - - axes = list(data['domain']['axes'].keys()) - axes.sort() - assert len(axes) == 3 - assert axes == ['TIME', 'x', 'y'] - - assert data['domain']['axes']['x']['start'] == 11.0 - assert data['domain']['axes']['x']['stop'] == 11.0 - assert data['domain']['axes']['y']['start'] == 11.0 - assert data['domain']['axes']['y']['stop'] == 11.0 - - parameters = list(data['parameters'].keys()) - parameters.sort() - assert len(parameters) == 4 - assert parameters == ['AIRT', 'SST', 'UWND', 'VWND'] - - # single parameter - req = mock_request({ - 'coords': 'POINT(11 11)', 'parameter_names': 'SST' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - - assert len(data['parameters'].keys()) == 1 - assert list(data['parameters'].keys())[0] == 'SST' - - # Zulu time zone - req = mock_request({ - 'coords': 'POINT(11 11)', - 'datetime': '2000-01-17T00:00:00Z/2000-06-16T23:00:00Z' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - # bounded date range - req = mock_request({ - 'coords': 'POINT(11 11)', - 'datetime': '2000-01-17/2000-06-16' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - time_dict = data['domain']['axes']['TIME'] - - assert time_dict['start'] == '2000-02-15T16:29:05.999999999' - assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' - assert time_dict['num'] == 5 - - # unbounded date range - start - req = mock_request({ - 'coords': 'POINT(11 11)', - 'datetime': '../2000-06-16' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - time_dict = data['domain']['axes']['TIME'] - - assert time_dict['start'] == '2000-01-16T06:00:00.000000000' - assert time_dict['stop'] == '2000-06-16T10:25:30.000000000' - assert time_dict['num'] == 6 - - # unbounded date range - end - req = mock_request({ - 'coords': 'POINT(11 11)', - 'datetime': '2000-06-16/..' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - data = json.loads(response) - time_dict = data['domain']['axes']['TIME'] - - assert time_dict['start'] == '2000-06-16T10:25:30.000000000' - assert time_dict['stop'] == '2000-12-16T01:20:05.999999996' - assert time_dict['num'] == 7 - - # some data - req = mock_request({ - 'coords': 'POINT(11 11)', 'datetime': '2000-01-16' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.OK - - # no data - req = mock_request({ - 'coords': 'POINT(11 11)', 'datetime': '2000-01-17' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.NO_CONTENT - - # position no coords - req = mock_request({ - 'datetime': '2000-01-17' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'position') - assert code == HTTPStatus.BAD_REQUEST - - # cube bbox parameter 4 dimensional - req = mock_request({ - 'bbox': '0,0,10,10' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'cube') - assert code == HTTPStatus.OK - - # cube bad bbox parameter - req = mock_request({ - 'bbox': '0,0,10' - }) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'cube') - assert code == HTTPStatus.BAD_REQUEST - - # cube no bbox parameter - req = mock_request({}) - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'icoads-sst', None, 'cube') - assert code == HTTPStatus.BAD_REQUEST - - # cube decreasing latitude coords and S3 - req = mock_request({ - 'bbox': '-100,40,-99,45', - 'parameter_names': 'tmn', - 'datetime': '1994-01-01/1994-12-31', - }) - - rsp_headers, code, response = api_.get_collection_edr_query( - req, 'usgs-prism', None, 'cube') - assert code == HTTPStatus.OK - - -def test_validate_bbox(): - assert validate_bbox('1,2,3,4') == [1, 2, 3, 4] - assert validate_bbox('1,2,3,4,5,6') == [1, 2, 3, 4, 5, 6] - assert validate_bbox('-142,42,-52,84') == [-142, 42, -52, 84] - assert (validate_bbox('-142.1,42.12,-52.22,84.4') == - [-142.1, 42.12, -52.22, 84.4]) - assert (validate_bbox('-142.1,42.12,-5.28,-52.22,84.4,7.39') == - [-142.1, 42.12, -5.28, -52.22, 84.4, 7.39]) - - assert (validate_bbox('177.0,65.0,-177.0,70.0') == - [177.0, 65.0, -177.0, 70.0]) - - with pytest.raises(ValueError): - validate_bbox('1,2,4') - - with pytest.raises(ValueError): - validate_bbox('1,2,4,5,6') - - with pytest.raises(ValueError): - validate_bbox('3,4,1,2') - - with pytest.raises(ValueError): - validate_bbox('1,2,6,4,5,3') - - -def test_validate_datetime(): - config = yaml_load(''' - temporal: - begin: 2000-10-30T18:24:39Z - end: 2007-10-30T08:57:29Z - ''') - - # test time instant - assert validate_datetime(config, '2004') == '2004' - assert validate_datetime(config, '2004-10') == '2004-10' - assert validate_datetime(config, '2001-10-30') == '2001-10-30' - - with pytest.raises(ValueError): - _ = validate_datetime(config, '2009-10-30') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2000-09-09') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2000-10-30T17:24:39Z') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2007-10-30T08:58:29Z') - - # test time envelope - assert validate_datetime(config, '2004/2005') == '2004/2005' - assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' - assert (validate_datetime(config, '2001-10-30/2002-10-30') == - '2001-10-30/2002-10-30') - assert validate_datetime(config, '2004/..') == '2004/..' - assert validate_datetime(config, '../2005') == '../2005' - assert validate_datetime(config, '2004/') == '2004/..' - assert validate_datetime(config, '/2005') == '../2005' - assert validate_datetime(config, '2004-10/2005-10') == '2004-10/2005-10' - assert (validate_datetime(config, '2001-10-30/2002-10-30') == - '2001-10-30/2002-10-30') - - with pytest.raises(ValueError): - _ = validate_datetime(config, '2007-11-01/..') - with pytest.raises(ValueError): - _ = validate_datetime(config, '2009/..') - with pytest.raises(ValueError): - _ = validate_datetime(config, '../2000-09') - with pytest.raises(ValueError): - _ = validate_datetime(config, '../1999') - - -@pytest.mark.parametrize("value, expected", [ - ('time(2000-11-11)', {'time': ['2000-11-11']}), - ('time("2000-11-11")', {'time': ['2000-11-11']}), - ('time("2000-11-11T00:11:11")', {'time': ['2000-11-11T00:11:11']}), - ('time("2000-11-11T11:12:13":"2021-12-22T:13:33:33")', {'time': ['2000-11-11T11:12:13', '2021-12-22T:13:33:33']}), # noqa - ('lat(40)', {'lat': [40]}), - ('lat(0:40)', {'lat': [0, 40]}), - ('foo("bar")', {'foo': ['bar']}), - ('foo("bar":"baz")', {'foo': ['bar', 'baz']}) -]) -def test_validate_subset(value, expected): - assert validate_subset(value) == expected - - with pytest.raises(ValueError): - validate_subset('foo("bar)') - - -def test_get_exception(config, api_): - d = api_.get_exception(500, {}, 'json', 'NoApplicableCode', 'oops') - assert d[0] == {} - assert d[1] == 500 - content = json.loads(d[2]) - assert content['code'] == 'NoApplicableCode' - assert content['description'] == 'oops' - - d = api_.get_exception(500, {}, 'html', 'NoApplicableCode', 'oops') - assert d[0] == {'Content-Type': 'text/html'} diff --git a/tests/test_api_ogr_provider.py b/tests/test_api_ogr_provider.py index 65d3587..c29fc62 100644 --- a/tests/test_api_ogr_provider.py +++ b/tests/test_api_ogr_provider.py @@ -4,7 +4,7 @@ # Authors: Tom Kralidis # # Copyright (c) 2019 Just van den Broecke -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -35,8 +35,10 @@ import logging import pytest from pygeoapi.api import API +from pygeoapi.api.itemtypes import get_collection_item, get_collection_items from pygeoapi.util import yaml_load, geojson_to_geom -from .util import get_test_file_path, mock_request + +from .util import get_test_file_path, mock_api_request LOGGER = logging.getLogger(__name__) @@ -70,16 +72,16 @@ def test_get_collection_items_bbox_crs(config, api_): COLLECTIONS = ['dutch_addresses_4326', 'dutch_addresses_28992'] for coll in COLLECTIONS: # bbox-crs full extent - req = mock_request({'bbox': '5.670670, 52.042700, 5.829110, 52.123700', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '5.670670, 52.042700, 5.829110, 52.123700', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 # bbox-crs partial extent, 1 feature, request with multiple CRSs for crs in CRS_BBOX_DICT: - req = mock_request({'bbox': CRS_BBOX_DICT[crs], 'bbox-crs': crs}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': CRS_BBOX_DICT[crs], 'bbox-crs': crs}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 1 @@ -88,29 +90,29 @@ def test_get_collection_items_bbox_crs(config, api_): assert properties['huisnummer'] == '2' # bbox-crs outside extent - req = mock_request({'bbox': '5, 51.9, 5.1, 52.0', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '5, 51.9, 5.1, 52.0', 'bbox-crs': 'http://www.opengis.net/def/crs/OGC/1.3/CRS84'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 0 # bbox-crs outside extent - req = mock_request({'bbox': '130000, 440000, 140000, 450000', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/28992'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '130000, 440000, 140000, 450000', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/28992'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 0 # bbox-crs outside extent - axis reversed CRS - req = mock_request({'bbox': '51.9, 5, 52.0, 5.1', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '51.9, 5, 52.0, 5.1', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 0 # bbox-crs full extent - axis reversed CRS - req = mock_request({'bbox': '52.042700, 5.670670, 52.123700, 5.829110', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({'bbox': '52.042700, 5.670670, 52.123700, 5.829110', 'bbox-crs': 'http://www.opengis.net/def/crs/EPSG/0/4326'}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 @@ -135,8 +137,8 @@ def test_get_collection_items_crs(config, api_): COLLECTIONS = ['dutch_addresses_4326', 'dutch_addresses_28992'] for coll in COLLECTIONS: # crs full extent to get target feature - req = mock_request({}) # noqa - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + req = mock_api_request({}) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 @@ -145,13 +147,13 @@ def test_get_collection_items_crs(config, api_): # request with multiple CRSs for crs in CRS_DICT: # Do for query (/items) - req = mock_request({'crs': crs}) # noqa + req = mock_api_request({'crs': crs}) # noqa if crs == 'none': # Test for default bbox CRS - req = mock_request({}) # noqa + req = mock_api_request({}) # noqa crs = DEFAULT_CRS - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 10 @@ -173,8 +175,8 @@ def test_get_collection_items_crs(config, api_): assert test_geom.equals_exact(crs_geom, 1), f'coords not equal for CRS: {crs} {crs_geom} in COLL: {coll} {test_geom}' # noqa # Same for single Feature 'get' - req = mock_request({'crs': crs}) # noqa - rsp_headers, code, response = api_.get_collection_item(req, coll, feature_id) # noqa + req = mock_api_request({'crs': crs}) # noqa + rsp_headers, code, response = get_collection_item(api_, req, coll, feature_id) # noqa test_feature = json.loads(response) assert test_feature['id'] == feature_id @@ -195,13 +197,13 @@ def test_get_collection_items_crs(config, api_): # Test combining BBOX and BBOX-CRS for bbox_crs in CRS_BBOX_DICT: # Do for query (/items) - req = mock_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs], 'bbox-crs': bbox_crs}) # noqa + req = mock_api_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs], 'bbox-crs': bbox_crs}) # noqa if bbox_crs == 'none': # Test for default bbox CRS - req = mock_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs]}) # noqa + req = mock_api_request({'crs': crs, 'bbox': CRS_BBOX_DICT[bbox_crs]}) # noqa bbox_crs = DEFAULT_CRS - rsp_headers, code, response = api_.get_collection_items(req, coll) # noqa + rsp_headers, code, response = get_collection_items(api_, req, coll) # noqa features = json.loads(response)['features'] assert len(features) == 1 diff --git a/tests/test_postgresql_provider.py b/tests/test_postgresql_provider.py index 650a609..b027d08 100644 --- a/tests/test_postgresql_provider.py +++ b/tests/test_postgresql_provider.py @@ -7,7 +7,7 @@ # Francesco Bartoli # # Copyright (c) 2019 Just van den Broecke -# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2024 Tom Kralidis # Copyright (c) 2022 John A Stevenson and Colin Blackburn # Copyright (c) 2023 Francesco Bartoli # @@ -47,7 +47,9 @@ from http import HTTPStatus from pygeofilter.parsers.ecql import parse from pygeoapi.api import API - +from pygeoapi.api.itemtypes import ( + get_collection_items, get_collection_item, post_collection_items +) from pygeoapi.provider.base import ( ProviderConnectionError, ProviderItemNotFoundError, @@ -59,7 +61,7 @@ import pygeoapi.provider.postgresql as postgresql_provider_module from pygeoapi.util import (yaml_load, geojson_to_geom, get_transform_from_crs, get_crs_from_uri) -from .util import get_test_file_path, mock_request +from .util import get_test_file_path, mock_api_request PASSWORD = os.environ.get('POSTGRESQL_PASSWORD', 'postgres') DEFAULT_CRS = 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' @@ -429,12 +431,12 @@ def test_get_collection_items_postgresql_cql(pg_api_): expected_ids = [80835474, 80835483] # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-text', 'filter': cql_query }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.OK @@ -443,11 +445,11 @@ def test_get_collection_items_postgresql_cql(pg_api_): assert ids == expected_ids # Act, no filter-lang - req = mock_request({ + req = mock_api_request({ 'filter': cql_query }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.OK @@ -467,12 +469,12 @@ def test_get_collection_items_postgresql_cql_invalid_filter_language(pg_api_): cql_query = 'osm_id BETWEEN 80800000 AND 80900000 AND name IS NULL' # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-json', # Only cql-text is valid for GET 'filter': cql_query }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -494,11 +496,11 @@ def test_get_collection_items_postgresql_cql_bad_cql(pg_api_, bad_cql): Test for bad cql """ # Act - req = mock_request({ + req = mock_api_request({ 'filter': bad_cql }) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -524,11 +526,11 @@ def test_post_collection_items_postgresql_cql(pg_api_): expected_ids = [80835474, 80835483] # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-json' }, data=cql, **headers) - rsp_headers, code, response = pg_api_.post_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = post_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.OK @@ -550,11 +552,11 @@ def test_post_collection_items_postgresql_cql_invalid_filter_language(pg_api_): headers = {'CONTENT_TYPE': 'application/query-cql-json'} # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-text' # Only cql-json is valid for POST }, data=cql, **headers) - rsp_headers, code, response = pg_api_.post_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = post_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -580,11 +582,11 @@ def test_post_collection_items_postgresql_cql_bad_cql(pg_api_, bad_cql): headers = {'CONTENT_TYPE': 'application/query-cql-json'} # Act - req = mock_request({ + req = mock_api_request({ 'filter-lang': 'cql-json' }, data=bad_cql, **headers) - rsp_headers, code, response = pg_api_.post_collection_items( - req, 'hot_osm_waterways') + rsp_headers, code, response = post_collection_items( + pg_api_, req, 'hot_osm_waterways') # Assert assert code == HTTPStatus.BAD_REQUEST @@ -601,10 +603,9 @@ def test_get_collection_items_postgresql_crs(pg_api_): crs_32735 = 'http://www.opengis.net/def/crs/EPSG/0/32735' # Without CRS query parameter -> no coordinates transformation - req = mock_request({'bbox': '29.0,-2.85,29.05,-2.8'}) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways', - ) + req = mock_api_request({'bbox': '29.0,-2.85,29.05,-2.8'}) + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') assert code == HTTPStatus.OK @@ -613,10 +614,10 @@ def test_get_collection_items_postgresql_crs(pg_api_): # With CRS query parameter not resulting in coordinates transformation # (i.e. 'crs' query parameter is the same as 'storage_crs') - req = mock_request({'crs': storage_crs, 'bbox': '29.0,-2.85,29.05,-2.8'}) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways', - ) + req = mock_api_request( + {'crs': storage_crs, 'bbox': '29.0,-2.85,29.05,-2.8'}) + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' @@ -624,10 +625,9 @@ def test_get_collection_items_postgresql_crs(pg_api_): features_storage_crs = json.loads(response) # With CRS query parameter resulting in coordinates transformation - req = mock_request({'crs': crs_32735, 'bbox': '29.0,-2.85,29.05,-2.8'}) - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'hot_osm_waterways', - ) + req = mock_api_request({'crs': crs_32735, 'bbox': '29.0,-2.85,29.05,-2.8'}) + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'hot_osm_waterways') assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{crs_32735}>' @@ -688,10 +688,9 @@ def test_get_collection_item_postgresql_crs(pg_api_): ] for fid in fid_list: # Without CRS query parameter -> no coordinates transformation - req = mock_request({'f': 'json'}) - rsp_headers, code, response = pg_api_.get_collection_item( - req, 'hot_osm_waterways', fid, - ) + req = mock_api_request({'f': 'json'}) + rsp_headers, code, response = get_collection_item( + pg_api_, req, 'hot_osm_waterways', fid) assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{DEFAULT_CRS}>' @@ -701,10 +700,9 @@ def test_get_collection_item_postgresql_crs(pg_api_): # With CRS query parameter not resulting in coordinates transformation # (i.e. 'crs' query parameter is the same as 'storage_crs') - req = mock_request({'f': 'json', 'crs': storage_crs}) - rsp_headers, code, response = pg_api_.get_collection_item( - req, 'hot_osm_waterways', fid, - ) + req = mock_api_request({'f': 'json', 'crs': storage_crs}) + rsp_headers, code, response = get_collection_item( + pg_api_, req, 'hot_osm_waterways', fid) assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{storage_crs}>' @@ -716,10 +714,9 @@ def test_get_collection_item_postgresql_crs(pg_api_): assert feat_orig['geometry'] == feat_storage_crs['geometry'] # With CRS query parameter resulting in coordinates transformation - req = mock_request({'f': 'json', 'crs': crs_32735}) - rsp_headers, code, response = pg_api_.get_collection_item( - req, 'hot_osm_waterways', fid, - ) + req = mock_api_request({'f': 'json', 'crs': crs_32735}) + rsp_headers, code, response = get_collection_item( + pg_api_, req, 'hot_osm_waterways', fid) assert code == HTTPStatus.OK assert rsp_headers['Content-Crs'] == f'<{crs_32735}>' @@ -741,9 +738,9 @@ def test_get_collection_items_postgresql_automap_naming_conflicts(pg_api_): Test that PostgreSQLProvider can handle naming conflicts when automapping classes and relationships from database schema. """ - req = mock_request() - rsp_headers, code, response = pg_api_.get_collection_items( - req, 'dummy_naming_conflicts') + req = mock_api_request() + rsp_headers, code, response = get_collection_items( + pg_api_, req, 'dummy_naming_conflicts') assert code == HTTPStatus.OK features = json.loads(response).get('features') diff --git a/tests/util.py b/tests/util.py index 5de9745..64b0c37 100644 --- a/tests/util.py +++ b/tests/util.py @@ -40,6 +40,8 @@ from werkzeug.test import create_environ from werkzeug.wrappers import Request from werkzeug.datastructures import ImmutableMultiDict +from pygeoapi.api import APIRequest + LOGGER = logging.getLogger(__name__) @@ -77,6 +79,25 @@ def mock_request(params: dict = None, data=None, **headers) -> Request: return request +def mock_api_request(params: dict | None = None, data=None, **headers + ) -> APIRequest: + """ + Mocks an APIRequest + + :param params: Optional query parameter dict for the request. + Will be set to {} if omitted. + :param data: Optional data/body to send with the request. + Can be text/bytes or a JSON dictionary. + :param headers: Optional request HTTP headers to set. + :returns: APIRequest instance + """ + return APIRequest.from_flask( + mock_request(params=params, data=data, **headers), + # NOTE: could also read supported_locales from test config + supported_locales=['en-US', 'fr-CA'], + ) + + @contextmanager def mock_flask(config_file: str = 'pygeoapi-test-config.yml', openapi_file: str = 'pygeoapi-test-openapi.yml',