From 8e122d1a619f5a460b4bdc9b6dd3bb39baa31a39 Mon Sep 17 00:00:00 2001 From: Benjamin Webb <40066515+webb-ben@users.noreply.github.com> Date: Wed, 3 Jan 2024 08:45:07 -0700 Subject: [PATCH] Add Admin API (#1137) * Add Admin API - Create `admin.py` to serve as Admin API Core - Create `flask_admin.py` to create flask blueprint for admin API - Consolidate configuration getter - Add Pathlib serializing - Add docker example * Add integration tests - Amend admin example to allow writing to configuration. If FS is read only admin API does not work. Returns a 500 and logs `OSError: [Errno 30] Read-only file system: '/pygeoapi/local.config.yml' ` * Preserve env variables in configuration * Use common accessor functions - Use common configuration accessor methods for Django and Starlette * GET returns raw config file Return configuration with environment variables preserved on GET requests * Safeguard env variables for root cfg view "bind": { "host": "localhost", "port": "6000" } -> "bind": { "host": "${HOST}", "port": "${PORT}" } * Simplify admin HTML imports - Use jinja recursion to expand the configuration - Remove vue from templates * Create admin API documentation * Use render_item_value in admin template * Add Admin API - Create `admin.py` to serve as Admin API Core - Create `flask_admin.py` to create flask blueprint for admin API - Consolidate configuration getter - Add Pathlib serializing - Add docker example * Update GitHub Actions deployment * Update admin entrypoint Update admin entrypoint to align with upstream pygeoapi implementation * Make requested changes Co-Authored-By: Tom Kralidis * Amend test url Co-Authored-By: Tom Kralidis * Fix Admin CI tests * Add PUT and PATCH for root configuration - Add put and patch for root configuration - Add CI tests for PUT and PATCH of root * Update OpenAPI document wording * Update entrypoint.sh Replace tabs with spaces * Remove unused step Error from rebasing. Admin API tests are moved to their own job. * Use jsonpatch - Use debian supported packaging - Use custom merge function * Move test data location * Create Starlette and Django app - Fold flask_admin.py into flask_app.py Co-Authored-By: Tom Kralidis * Make requirements-admin.txt Move admin dependencies to requirements-admin.txt * Delete guiblock.html * Update test count for STAC Update expected test count for addt'l admin test data * Relegate config warning to config.py * Move admin tests out of example * Delete admin docker example * Update admin-api.rst * Update pygeoapi-config-0.x.yml * Update configuration.rst * Update config.py * Update admin.py * Update admin.py --------- Co-authored-by: Tom Kralidis --- .github/workflows/main.yml | 42 ++ Dockerfile | 2 + docs/source/admin-api.rst | 30 + docs/source/configuration.rst | 1 + docs/source/index.rst | 1 + pygeoapi/admin.py | 623 ++++++++++++++++++ pygeoapi/config.py | 28 +- pygeoapi/django_/settings.py | 4 +- pygeoapi/django_/urls.py | 22 + pygeoapi/django_/views.py | 66 +- pygeoapi/django_app.py | 14 +- pygeoapi/flask_app.py | 77 ++- pygeoapi/openapi.py | 230 ++++++- .../schemas/config/pygeoapi-config-0.x.yml | 5 + pygeoapi/starlette_app.py | 71 +- pygeoapi/templates/_base.html | 5 + pygeoapi/templates/admin/index.html | 59 ++ pygeoapi/util.py | 48 +- requirements-admin.txt | 3 + tests/data/admin/admin-patch.json | 5 + tests/data/admin/admin-put.json | 72 ++ tests/data/admin/resource-patch.json | 5 + tests/data/admin/resource-post.json | 48 ++ tests/data/admin/resource-put.json | 46 ++ tests/pygeoapi-test-config-admin.yml | 101 +++ tests/test_admin_api.py | 160 +++++ tests/test_filesystem_provider.py | 2 +- 27 files changed, 1719 insertions(+), 51 deletions(-) create mode 100644 docs/source/admin-api.rst create mode 100644 pygeoapi/admin.py create mode 100644 pygeoapi/templates/admin/index.html create mode 100644 requirements-admin.txt create mode 100644 tests/data/admin/admin-patch.json create mode 100644 tests/data/admin/admin-put.json create mode 100644 tests/data/admin/resource-patch.json create mode 100644 tests/data/admin/resource-post.json create mode 100644 tests/data/admin/resource-put.json create mode 100644 tests/pygeoapi-test-config-admin.yml create mode 100644 tests/test_admin_api.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 15272f5..fe936e8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -94,6 +94,7 @@ jobs: - name: Install requirements 📦 run: | pip3 install -r requirements.txt + pip3 install -r requirements-admin.txt pip3 install -r requirements-starlette.txt pip3 install -r requirements-dev.txt pip3 install -r requirements-provider.txt @@ -151,3 +152,44 @@ jobs: if: ${{ failure() }} run: | pip3 list -v + + admin: + needs: [flake8_py3] + runs-on: ubuntu-20.04 + strategy: + matrix: + include: + - python-version: 3.8 + env: + PYGEOAPI_CONFIG: "tests/pygeoapi-test-config-admin.yml" + PYGEOAPI_OPENAPI: "tests/pygeoapi-test-openapi-admin.yml" + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + name: Setup Python ${{ matrix.python-version }} + with: + python-version: ${{ matrix.python-version }} + - uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: gunicorn python3-gevent + version: 1.0 + - name: Install requirements 📦 + run: | + pip3 install -r requirements.txt + pip3 install -r requirements-dev.txt + pip3 install -r requirements-admin.txt + python3 setup.py install + - name: Run pygeoapi with admin API ⚙️ + run: | + pygeoapi openapi generate ${PYGEOAPI_CONFIG} --output-file ${PYGEOAPI_OPENAPI} + gunicorn --bind 0.0.0.0:5000 \ + --reload \ + --reload-extra-file ${PYGEOAPI_CONFIG} \ + pygeoapi.flask_app:APP & + - name: run integration tests ⚙️ + run: | + pytest tests/test_admin_api.py + - name: failed tests 🚩 + if: ${{ failure() }} + run: | + pip3 list -v diff --git a/Dockerfile b/Dockerfile index ebb0eb9..0985a8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,6 +64,7 @@ ARG ADD_DEB_PACKAGES="\ python3-elasticsearch \ python3-fiona \ python3-gdal \ + python3-jsonpatch \ python3-netcdf4 \ python3-pandas \ python3-psycopg2 \ @@ -121,6 +122,7 @@ RUN \ # Install remaining pygeoapi deps && pip3 install -r requirements-docker.txt \ + && pip3 install -r requirements-admin.txt \ # Install pygeoapi && pip3 install -e . \ diff --git a/docs/source/admin-api.rst b/docs/source/admin-api.rst new file mode 100644 index 0000000..920e88c --- /dev/null +++ b/docs/source/admin-api.rst @@ -0,0 +1,30 @@ +.. _admin-api: + +Admin API +========= + +pygeoapi provides the ability to manage configuration through an API. + +When enabled, :ref:`transactions` can be made on pygeoapi's configured resources. This allows for API based modification of the pygeoapi configuration. + +The API is enabled with the following server configuration: + +.. code-block:: yaml + + server: + admin: true # boolean on whether to enable Admin API. + +For pygeoapi to hot reload the configuration as changes are made, the pygeoapi configuration file must be included as +demonstrated for a gunicorn deployment of pygeoapi via flask: + +.. code-block:: bash + + gunicorn \ + --workers ${WSGI_WORKERS} \ + --worker-class=${WSGI_WORKER_CLASS} \ + --timeout ${WSGI_WORKER_TIMEOUT} \ + --name=${CONTAINER_NAME} \ + --bind ${CONTAINER_HOST}:${CONTAINER_PORT} \ + --reload \ + --reload-extra-file ${PYGEOAPI_CONFIG} \ + pygeoapi.flask_app:APP diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index 334a1f0..feca6ab 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -50,6 +50,7 @@ For more information related to API design rules (the ``api_rules`` property in cors: true # boolean on whether server should support CORS pretty_print: true # whether JSON responses should be pretty-printed limit: 10 # server limit on number of items to return + admin: false # whether to enable the Admin API templates: # optional configuration to specify a different set of templates for HTML pages. Recommend using absolute paths. Omit this to use the default provided templates path: /path/to/jinja2/templates/folder # path to templates folder containing the Jinja2 template HTML files diff --git a/docs/source/index.rst b/docs/source/index.rst index a539bd7..1b6ab82 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -39,6 +39,7 @@ reference documentation on all aspects of the project. openapi data-publishing/index transactions + admin-api plugins html-templating crs diff --git a/pygeoapi/admin.py b/pygeoapi/admin.py new file mode 100644 index 0000000..ff98b61 --- /dev/null +++ b/pygeoapi/admin.py @@ -0,0 +1,623 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# Benjamin Webb +# +# Copyright (c) 2023 Tom Kralidis +# Copyright (c) 2023 Benjamin Webb +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +from copy import deepcopy +import os +import json +from jsonpatch import make_patch +from jsonschema.exceptions import ValidationError +import logging +from typing import Any, Tuple, Union + +from pygeoapi.api import API, APIRequest, F_HTML, pre_process + +from pygeoapi.config import get_config, validate_config +from pygeoapi.openapi import get_oas +# from pygeoapi.openapi import validate_openapi_document +from pygeoapi.util import to_json, render_j2_template, yaml_dump + + +LOGGER = logging.getLogger(__name__) + + +class Admin(API): + """Admin object""" + + PYGEOAPI_CONFIG = os.environ.get('PYGEOAPI_CONFIG') + PYGEOAPI_OPENAPI = os.environ.get('PYGEOAPI_OPENAPI') + + def __init__(self, config, openapi): + """ + constructor + + :param config: configuration dict + :param openapi: openapi dict + + :returns: `pygeoapi.Admin` instance + """ + + super().__init__(config, openapi) + + def merge(self, obj1, obj2): + """ + Merge two dictionaries + + :param obj1: `dict` of first object + :param obj2: `dict` of second object + + :returns: `dict` of merged objects + """ + + if isinstance(obj1, dict) and isinstance(obj2, dict): + merged = obj1.copy() + for key, value in obj2.items(): + if key in merged: + merged[key] = self.merge(merged[key], value) + else: + merged[key] = value + return merged + elif isinstance(obj1, list) and isinstance(obj2, list): + return [self.merge(i1, i2) for i1, i2 in zip(obj1, obj2)] + else: + return obj2 + + def validate(self, config): + """ + Validate pygeoapi configuration and OpenAPI to file + + :param config: configuration dict + """ + + # validate pygeoapi configuration + LOGGER.debug('Validating configuration') + validate_config(config) + # validate OpenAPI document + # LOGGER.debug('Validating openapi document') + # oas = get_oas(config) + # validate_openapi_document(oas) + return True + + def write(self, config): + """ + Write pygeoapi configuration and OpenAPI to file + + :param config: configuration dict + """ + + self.write_config(config) + self.write_oas(config) + + def write_config(self, config): + """ + Write pygeoapi configuration file + + :param config: configuration dict + """ + + # validate pygeoapi configuration + config = deepcopy(config) + validate_config(config) + + # Preserve env variables + LOGGER.debug('Reading env variables in configuration') + raw_conf = get_config(raw=True) + conf = get_config() + patch = make_patch(conf, raw_conf) + + LOGGER.debug('Merging env variables') + config = patch.apply(config) + + # write pygeoapi configuration + LOGGER.debug('Writing pygeoapi configutation') + yaml_dump(config, self.PYGEOAPI_CONFIG) + LOGGER.debug('Finished writing pygeoapi configuration') + + def write_oas(self, config): + """ + Write pygeoapi OpenAPI document + + :param config: configuration dict + """ + + # validate OpenAPI document + config = deepcopy(config) + oas = get_oas(config) + # validate_openapi_document(oas) + + # write OpenAPI document + LOGGER.debug('Writing OpenAPI document') + yaml_dump(oas, self.PYGEOAPI_OPENAPI) + LOGGER.debug('Finished writing OpenAPI document') + + @pre_process + def get_config( + self, + request: Union[APIRequest, Any] + ) -> Tuple[dict, int, str]: + """ + Provide admin configuration document + + :param request: request object + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + headers = request.get_response_headers() + + cfg = get_config(raw=True) + + if request.format == F_HTML: + content = render_j2_template( + self.config, 'admin/index.html', cfg, request.locale + ) + else: + content = to_json(cfg, self.pretty_print) + + return headers, 200, content + + @pre_process + def put_config( + self, + request: Union[APIRequest, Any] + ) -> Tuple[dict, int, str]: + """ + Update complete pygeoapi configuration + + :param request: request object + + :returns: tuple of headers, status code, content + """ + + LOGGER.debug('Updating configuration') + + if not request.is_valid(): + return self.get_format_exception(request) + + headers = request.get_response_headers() + + data = request.data + if not data: + msg = 'missing request data' + return self.get_exception( + 400, headers, request.format, 'MissingParameterValue', msg + ) + + try: + # Parse data + data = data.decode() + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input is not valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return self.get_exception( + 400, headers, request.format, 'InvalidParameterValue', msg + ) + + LOGGER.debug('Updating configuration') + try: + self.validate(data) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return self.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) + + self.write(data) + + return headers, 204, {} + + @pre_process + def patch_config( + self, request: Union[APIRequest, Any] + ) -> Tuple[dict, int, str]: + """ + Update partial pygeoapi configuration + + :param request: request object + :param resource_id: resource identifier + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + config = deepcopy(self.config) + headers = request.get_response_headers() + + data = request.data + if not data: + msg = 'missing request data' + return self.get_exception( + 400, headers, request.format, 'MissingParameterValue', msg + ) + + try: + # Parse data + data = data.decode() + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input is not valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return self.get_exception( + 400, headers, request.format, 'InvalidParameterValue', msg + ) + + LOGGER.debug('Merging configuration') + config = self.merge(config, data) + + try: + self.validate(config) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return self.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) + + self.write(config) + + content = to_json(config, self.pretty_print) + + return headers, 204, content + + @pre_process + def get_resources( + self, request: Union[APIRequest, Any] + ) -> Tuple[dict, int, str]: + """ + Provide admin document + + :param request: request object + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + headers = request.get_response_headers() + + cfg = get_config(raw=True) + + if request.format == F_HTML: + content = render_j2_template( + self.config, + 'admin/index.html', + cfg['resources'], + request.locale, + ) + else: + content = to_json(cfg['resources'], self.pretty_print) + + return headers, 200, content + + @pre_process + def post_resource( + self, request: Union[APIRequest, Any] + ) -> Tuple[dict, int, str]: + """ + Add resource configuration + + :param request: request object + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + config = deepcopy(self.config) + headers = request.get_response_headers() + + data = request.data + if not data: + msg = 'missing request data' + return self.get_exception( + 400, headers, request.format, 'MissingParameterValue', msg + ) + + try: + # Parse data + data = data.decode() + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input is not valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return self.get_exception( + 400, headers, request.format, 'InvalidParameterValue', msg + ) + + resource_id = next(iter(data.keys())) + + if config['resources'].get(resource_id) is not None: + # Resource already exists + msg = f'Resource exists: {resource_id}' + LOGGER.error(msg) + return self.get_exception( + 400, headers, request.format, 'NoApplicableCode', msg + ) + + LOGGER.debug(f'Adding resource: {resource_id}') + config['resources'].update(data) + + try: + self.validate(config) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return self.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) + + self.write(config) + + content = f'Location: /{request.path_info}/{resource_id}' + LOGGER.debug(f'Success at {content}') + + return headers, 201, content + + @pre_process + def get_resource( + self, request: Union[APIRequest, Any], resource_id: str + ) -> Tuple[dict, int, str]: + """ + Get resource configuration + + :param request: request object + :param resource_id: + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + headers = request.get_response_headers() + + cfg = get_config(raw=True) + + try: + resource = cfg['resources'][resource_id] + except KeyError: + msg = f'Resource not found: {resource_id}' + return self.get_exception( + 400, headers, request.format, 'ResourceNotFound', msg + ) + + if request.format == F_HTML: + content = render_j2_template( + self.config, 'admin/index.html', resource, request.locale + ) + else: + content = to_json(resource, self.pretty_print) + + return headers, 200, content + + @pre_process + def delete_resource( + self, request: Union[APIRequest, Any], resource_id: str + ) -> Tuple[dict, int, str]: + """ + Delete resource configuration + + :param request: request object + :param resource_id: resource identifier + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + config = deepcopy(self.config) + headers = request.get_response_headers() + + try: + LOGGER.debug(f'Removing resource configuration for: {resource_id}') + config['resources'].pop(resource_id) + except KeyError: + msg = f'Resource not found: {resource_id}' + return self.get_exception( + 400, headers, request.format, 'ResourceNotFound', msg + ) + + LOGGER.debug('Resource removed, validating and saving configuration') + try: + self.validate(config) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return self.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) + + self.write(config) + + return headers, 204, {} + + @pre_process + def put_resource( + self, + request: Union[APIRequest, Any], + resource_id: str, + ) -> Tuple[dict, int, str]: + """ + Update complete resource configuration + + :param request: request object + :param resource_id: resource identifier + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + config = deepcopy(self.config) + headers = request.get_response_headers() + + try: + LOGGER.debug('Verifying resource exists') + config['resources'][resource_id] + except KeyError: + msg = f'Resource not found: {resource_id}' + return self.get_exception( + 400, headers, request.format, 'ResourceNotFound', msg + ) + + data = request.data + if not data: + msg = 'missing request data' + return self.get_exception( + 400, headers, request.format, 'MissingParameterValue', msg + ) + + try: + # Parse data + data = data.decode() + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input is not valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return self.get_exception( + 400, headers, request.format, 'InvalidParameterValue', msg + ) + + LOGGER.debug(f'Updating resource: {resource_id}') + config['resources'].update({resource_id: data}) + try: + self.validate(config) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return self.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) + + self.write(config) + + return headers, 204, {} + + @pre_process + def patch_resource( + self, request: Union[APIRequest, Any], resource_id: str + ) -> Tuple[dict, int, str]: + """ + Update partial resource configuration + + :param request: request object + :param resource_id: resource identifier + + :returns: tuple of headers, status code, content + """ + + if not request.is_valid(): + return self.get_format_exception(request) + + config = deepcopy(self.config) + headers = request.get_response_headers() + + try: + LOGGER.debug('Verifying resource exists') + resource = config['resources'][resource_id] + except KeyError: + msg = f'Resource not found: {resource_id}' + return self.get_exception( + 400, headers, request.format, 'ResourceNotFound', msg + ) + + data = request.data + if not data: + msg = 'missing request data' + return self.get_exception( + 400, headers, request.format, 'MissingParameterValue', msg + ) + + try: + # Parse data + data = data.decode() + except (UnicodeDecodeError, AttributeError): + pass + + try: + data = json.loads(data) + except (json.decoder.JSONDecodeError, TypeError) as err: + # Input is not valid JSON + LOGGER.error(err) + msg = 'invalid request data' + return self.get_exception( + 400, headers, request.format, 'InvalidParameterValue', msg + ) + + LOGGER.debug('Merging resource block') + data = self.merge(resource, data) + LOGGER.debug('Updating resource') + config['resources'].update({resource_id: data}) + + try: + self.validate(config) + except ValidationError as err: + LOGGER.error(err) + msg = 'Schema validation error' + return self.get_exception( + 400, headers, request.format, 'ValidationError', msg + ) + + self.write(config) + + content = to_json(resource, self.pretty_print) + + return headers, 204, content diff --git a/pygeoapi/config.py b/pygeoapi/config.py index b5dae66..07d8615 100644 --- a/pygeoapi/config.py +++ b/pygeoapi/config.py @@ -33,17 +33,38 @@ import click import json from jsonschema import validate as jsonschema_validate import logging -from pathlib import Path +import os +import yaml -from pygeoapi.util import to_json, yaml_load +from pygeoapi.util import to_json, yaml_load, THISDIR LOGGER = logging.getLogger(__name__) -THISDIR = Path(__file__).parent.resolve() + +def get_config(raw: bool = False) -> dict: + """ + Get pygeoapi configurations + + :param raw: `bool` over interpolation during config loading + + :returns: `dict` of pygeoapi configuration + """ + + if not os.environ.get('PYGEOAPI_CONFIG'): + raise RuntimeError('PYGEOAPI_CONFIG environment variable not set') + + with open(os.environ.get('PYGEOAPI_CONFIG'), encoding='utf8') as fh: + if raw: + CONFIG = yaml.safe_load(fh) + else: + CONFIG = yaml_load(fh) + + return CONFIG def load_schema() -> dict: """ Reads the JSON schema YAML file. """ + schema_file = THISDIR / 'schemas' / 'config' / 'pygeoapi-config-0.x.yml' with schema_file.open() as fh2: @@ -58,6 +79,7 @@ def validate_config(instance_dict: dict) -> bool: :returns: `bool` of validation """ + jsonschema_validate(json.loads(to_json(instance_dict)), load_schema()) return True diff --git a/pygeoapi/django_/settings.py b/pygeoapi/django_/settings.py index 1185c7c..ab5776c 100644 --- a/pygeoapi/django_/settings.py +++ b/pygeoapi/django_/settings.py @@ -47,7 +47,7 @@ https://docs.djangoproject.com/en/2.2/ref/settings/ import os # pygeoapi specific -from pygeoapi.django_app import config +from pygeoapi.config import get_config from pygeoapi.util import get_api_rules # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -166,7 +166,7 @@ STATIC_ROOT = os.path.join(BASE_DIR, 'assets') STATIC_URL = '/static/' # pygeoapi specific -PYGEOAPI_CONFIG = config() +PYGEOAPI_CONFIG = get_config() API_RULES = get_api_rules(PYGEOAPI_CONFIG) diff --git a/pygeoapi/django_/urls.py b/pygeoapi/django_/urls.py index 0ca6b87..cc72091 100644 --- a/pygeoapi/django_/urls.py +++ b/pygeoapi/django_/urls.py @@ -241,6 +241,28 @@ if url_route_prefix: path(url_route_prefix, include(urlpatterns)) ] +if settings.PYGEOAPI_CONFIG['server'].get('admin', False): + admin_urlpatterns = [ + path( + apply_slash_rule('admin/config'), + views.admin_config, + name='admin-config' + ), + path( + apply_slash_rule('admin/config/resources'), + views.admin_config_resources, + name='admin-config-resources' + ), + path( + apply_slash_rule('admin/config/resources/'), + views.admin_config_resource, + name='admin-config-resource' + ), + ] + + urlpatterns.extend(admin_urlpatterns) + + # Add static URL and optionally add prefix (note: do NOT use django style here) url_static_prefix = settings.API_RULES.get_url_prefix() urlpatterns += static( diff --git a/pygeoapi/django_/views.py b/pygeoapi/django_/views.py index a578c79..3e00a44 100644 --- a/pygeoapi/django_/views.py +++ b/pygeoapi/django_/views.py @@ -34,9 +34,12 @@ # ================================================================= """Integration module for Django""" + from typing import Tuple, Dict, Mapping, Optional from django.conf import settings from django.http import HttpRequest, HttpResponse + +from pygeoapi.admin import Admin from pygeoapi.api import API @@ -485,12 +488,73 @@ def stac_catalog_search(request: HttpRequest) -> HttpResponse: pass +def admin_config(request: HttpRequest) -> HttpResponse: + """ + Admin landing page endpoint + + :returns: HTTP response + """ + + if request.method == 'GET': + return _feed_response(request, 'get_admin_config') + + elif request.method == 'PUT': + return _feed_response(request, 'put_admin_config') + + elif request.method == 'PATCH': + return _feed_response(request, 'patch_admin_config') + + +def admin_config_resources(request: HttpRequest) -> HttpResponse: + """ + Resource landing page endpoint + + :returns: HTTP response + """ + + if request.method == 'GET': + return _feed_response(request, 'get_admin_config_resources') + + elif request.method == 'POST': + return _feed_response(request, 'put_admin_config_resources') + + +def admin_config_resource(request: HttpRequest, + resource_id: str) -> HttpResponse: + """ + Resource landing page endpoint + + :returns: HTTP response + """ + + if request.method == 'GET': + return _feed_response(request, 'put_admin_config_resource', + resource_id) + + elif request.method == 'DELETE': + return _feed_response(request, 'delete_admin_config_resource', + resource_id) + + elif request.method == 'PUT': + return _feed_response(request, 'put_admin_config_resource', + resource_id) + + elif request.method == 'PATCH': + return _feed_response(request, 'patch_admin_config_resource', + resource_id) + + def _feed_response(request: HttpRequest, api_definition: str, *args, **kwargs) -> Tuple[Dict, int, str]: """Use pygeoapi api to process the input request""" - api_ = API(settings.PYGEOAPI_CONFIG) + if 'admin' not in api_definition: + api_ = API(settings.PYGEOAPI_CONFIG) + else: + api_ = Admin(settings.PYGEOAPI_CONFIG) + api = getattr(api_, api_definition) + return api(request, *args, **kwargs) diff --git a/pygeoapi/django_app.py b/pygeoapi/django_app.py index 8c922e2..57d601b 100644 --- a/pygeoapi/django_app.py +++ b/pygeoapi/django_app.py @@ -37,17 +37,7 @@ import os from pathlib import Path import sys - -def config(): - from pygeoapi.util import yaml_load - - if not os.environ.get('PYGEOAPI_CONFIG'): - raise RuntimeError('PYGEOAPI_CONFIG environment variable not set') - - with open(os.environ.get('PYGEOAPI_CONFIG'), encoding='utf8') as fh: - CONFIG = yaml_load(fh) - - return CONFIG +from pygeoapi.config import get_config def main(): @@ -62,7 +52,7 @@ def main(): 'forget to activate a virtual environment?' ) from exc - CONFIG = config() + CONFIG = get_config() bind = f"{CONFIG['server']['bind']['host']}:{CONFIG['server']['bind']['port']}" # noqa diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index f41cc7b..f2fceaa 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -3,7 +3,7 @@ # Authors: Tom Kralidis # Norman Barker # -# Copyright (c) 2022 Tom Kralidis +# Copyright (c) 2023 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -36,20 +36,14 @@ import click from flask import Flask, Blueprint, make_response, request, send_from_directory +from pygeoapi.admin import Admin from pygeoapi.api import API from pygeoapi.openapi import load_openapi_document -from pygeoapi.util import get_mimetype, yaml_load, get_api_rules +from pygeoapi.config import get_config +from pygeoapi.util import get_mimetype, get_api_rules -if 'PYGEOAPI_CONFIG' not in os.environ: - raise RuntimeError('PYGEOAPI_CONFIG environment variable not set') - -with open(os.environ.get('PYGEOAPI_CONFIG'), encoding='utf8') as fh: - CONFIG = yaml_load(fh) - -if 'PYGEOAPI_OPENAPI' not in os.environ: - raise RuntimeError('PYGEOAPI_OPENAPI environment variable not set') - +CONFIG = get_config() OPENAPI = load_openapi_document() API_RULES = get_api_rules(CONFIG) @@ -67,6 +61,7 @@ BLUEPRINT = Blueprint( static_folder=STATIC_FOLDER, url_prefix=API_RULES.get_url_prefix('flask') ) +ADMIN_BLUEPRINT = Blueprint('admin', __name__, static_folder=STATIC_FOLDER) # CORS: optionally enable from config. if CONFIG['server'].get('cors', False): @@ -464,8 +459,68 @@ def stac_catalog_path(path): return get_response(api_.get_stac_path(request, path)) +@ADMIN_BLUEPRINT.route('/admin/config', methods=['GET', 'PUT', 'PATCH']) +def admin_config(): + """ + Admin endpoint + + :returns: HTTP response + """ + + if request.method == 'GET': + return get_response(admin_.get_config(request)) + + elif request.method == 'PUT': + return get_response(admin_.put_config(request)) + + elif request.method == 'PATCH': + return get_response(admin_.patch_config(request)) + + +@ADMIN_BLUEPRINT.route('/admin/config/resources', methods=['GET', 'POST']) +def admin_config_resources(): + """ + Resources endpoint + + :returns: HTTP response + """ + + if request.method == 'GET': + return get_response(admin_.get_resources(request)) + + elif request.method == 'POST': + return get_response(admin_.post_resource(request)) + + +@ADMIN_BLUEPRINT.route( + '/admin/config/resources/', + methods=['GET', 'PUT', 'PATCH', 'DELETE']) +def admin_config_resource(resource_id): + """ + Resource endpoint + + :returns: HTTP response + """ + + if request.method == 'GET': + return get_response(admin_.get_resource(request, resource_id)) + + elif request.method == 'DELETE': + return get_response(admin_.delete_resource(request, resource_id)) + + elif request.method == 'PUT': + return get_response(admin_.put_resource(request, resource_id)) + + elif request.method == 'PATCH': + return get_response(admin_.patch_resource(request, resource_id)) + + APP.register_blueprint(BLUEPRINT) +if CONFIG['server'].get('admin'): + admin_ = Admin(CONFIG, OPENAPI) + APP.register_blueprint(ADMIN_BLUEPRINT) + @click.command() @click.pass_context diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 758d347..d697df2 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -59,11 +59,10 @@ OPENAPI_YAML = { 'oapif-2': 'https://schemas.opengis.net/ogcapi/features/part2/1.0/openapi/ogcapi-features-2.yaml', # noqa 'oapip': 'https://schemas.opengis.net/ogcapi/processes/part1/1.0/openapi', 'oacov': 'https://raw.githubusercontent.com/tomkralidis/ogcapi-coverages-1/fix-cis/yaml-unresolved', # noqa - 'oapit': 'https://raw.githubusercontent.com/opengeospatial/ogcapi-tiles/master/openapi/swaggerhub/tiles.yaml', # noqa - 'oapimt': 'https://raw.githubusercontent.com/opengeospatial/ogcapi-tiles/master/openapi/swaggerhub/map-tiles.yaml', # noqa 'oapir': 'https://raw.githubusercontent.com/opengeospatial/ogcapi-records/master/core/openapi', # noqa 'oaedr': 'https://schemas.opengis.net/ogcapi/edr/1.0/openapi', # noqa - 'oat': 'https://schemas.opengis.net/ogcapi/tiles/part1/1.0/openapi/ogcapi-tiles-1.yaml' # noqa + 'oapit': 'https://schemas.opengis.net/ogcapi/tiles/part1/1.0/openapi/ogcapi-tiles-1.yaml', # noqa + 'pygeoapi': 'https://raw.githubusercontent.com/geopython/pygeoapi/master/pygeoapi/schemas/config/pygeoapi-config-0.x.yml' # noqa } THISDIR = os.path.dirname(os.path.realpath(__file__)) @@ -432,6 +431,15 @@ def get_oas_30(cfg): 'additionalProperties': True }, 'style': 'form' + }, + 'resourceId': { + 'name': 'resourceId', + 'in': 'path', + 'description': 'Configuration resource identifier', + 'required': True, + 'schema': { + 'type': 'string' + } } }, 'schemas': { @@ -950,10 +958,10 @@ def get_oas_30(cfg): 'tags': [name], 'operationId': f'get{name.capitalize()}Tiles', 'parameters': [ - {'$ref': f"{OPENAPI_YAML['oat']}#/components/parameters/tileMatrixSetId"}, # noqa - {'$ref': f"{OPENAPI_YAML['oat']}#/components/parameters/tileMatrix"}, # noqa - {'$ref': f"{OPENAPI_YAML['oat']}#/components/parameters/tileRow"}, # noqa - {'$ref': f"{OPENAPI_YAML['oat']}#/components/parameters/tileCol"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrixSetId"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileMatrix"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileRow"}, # noqa + {'$ref': f"{OPENAPI_YAML['oapit']}#/components/parameters/tileCol"}, # noqa { 'name': 'f', 'in': 'query', @@ -1307,9 +1315,217 @@ def get_oas_30(cfg): oas['paths'] = paths + if cfg['server'].get('admin', False): + schema_dict = get_config_schema() + oas['definitions'] = schema_dict['definitions'] + LOGGER.debug('Adding admin endpoints') + oas['paths'].update(get_admin()) + return oas +def get_config_schema(): + schema_file = os.path.join(THISDIR, 'schemas', 'config', + 'pygeoapi-config-0.x.yml') + + with open(schema_file) as fh2: + return yaml_load(fh2) + + +def get_admin(): + + schema_dict = get_config_schema() + + paths = {} + + paths['/admin/config'] = { + 'get': { + 'summary': 'Get admin configuration', + 'description': 'Get admin configuration', + 'tags': ['admin'], + 'operationId': 'getAdminConfig', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': { + 'content': { + 'application/json': { + 'schema': schema_dict + } + } + } + } + }, + 'put': { + 'summary': 'Update admin configuration full', + 'description': 'Update admin configuration full', + 'tags': ['admin'], + 'operationId': 'putAdminConfig', + 'requestBody': { + 'description': 'Updates admin configuration', + 'content': { + 'application/json': { + 'schema': schema_dict + } + }, + 'required': True + }, + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + 'patch': { + 'summary': 'Partially update admin configuration', + 'description': 'Partially update admin configuration', + 'tags': ['admin'], + 'operationId': 'patchAdminConfig', + 'requestBody': { + 'description': 'Updates admin configuration', + 'content': { + 'application/json': { + 'schema': schema_dict + } + }, + 'required': True + }, + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + } + } + paths['/admin/config/resources'] = { + 'get': { + 'summary': 'Get admin configuration resources', + 'description': 'Get admin configuration resources', + 'tags': ['admin'], + 'operationId': 'getAdminConfigResources', + 'parameters': [ + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': { + 'content': { + 'application/json': { + 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa + } + } + } + } + }, + 'post': { + 'summary': 'Create admin configuration resource', + 'description': 'Create admin configuration resource', + 'tags': ['admin'], + 'operationId': 'postAdminConfigResource', + 'requestBody': { + 'description': 'Adds resource to configuration', + 'content': { + 'application/json': { + 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa + } + }, + 'required': True + }, + 'responses': { + '201': {'description': 'Successful creation'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + } + paths['/admin/config/resources/{resourceId}'] = { + 'get': { + 'summary': 'Get admin configuration resource', + 'description': 'Get admin configuration resource', + 'tags': ['admin'], + 'operationId': 'getAdminConfigResource', + 'parameters': [ + {'$ref': '#/components/parameters/resourceId'}, + {'$ref': '#/components/parameters/f'}, + {'$ref': '#/components/parameters/lang'} + ], + 'responses': { + '200': { + 'content': { + 'application/json': { + 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa + } + } + } + } + }, + 'put': { + 'summary': 'Update admin configuration resource', + 'description': 'Update admin configuration resource', + 'tags': ['admin'], + 'operationId': 'putAdminConfigResource', + 'parameters': [ + {'$ref': '#/components/parameters/resourceId'}, + ], + 'requestBody': { + 'description': 'Updates admin configuration resource', + 'content': { + 'application/json': { + 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa + } + }, + 'required': True + }, + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + 'patch': { + 'summary': 'Partially update admin configuration resource', + 'description': 'Partially update admin configuration resource', + 'tags': ['admin'], + 'operationId': 'patchAdminConfigResource', + 'parameters': [ + {'$ref': '#/components/parameters/resourceId'}, + ], + 'requestBody': { + 'description': 'Updates admin configuration resource', + 'content': { + 'application/json': { + 'schema': schema_dict['properties']['resources']['patternProperties']['^.*$'] # noqa + } + }, + 'required': True + }, + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '400': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/InvalidParameter"}, # noqa + '500': {'$ref': f"{OPENAPI_YAML['oapif-1']}#/components/responses/ServerError"} # noqa + } + }, + 'delete': { + 'summary': 'Delete admin configuration resource', + 'description': 'Delete admin configuration resource', + 'tags': ['admin'], + 'operationId': 'deleteAdminConfigResource', + 'parameters': [ + {'$ref': '#/components/parameters/resourceId'}, + ], + 'responses': { + '204': {'$ref': '#/components/responses/204'}, + '404': {'$ref': f"{OPENAPI_YAML['oapip']}/responses/NotFound.yaml"}, # noqa + 'default': {'$ref': '#/components/responses/default'} # noqa + } + } + } + + return paths + + def get_oas(cfg, version='3.0'): """ Stub to generate OpenAPI Document diff --git a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml index 8326425..5ba8ea8 100644 --- a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml +++ b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml @@ -25,6 +25,10 @@ properties: url: type: string description: URL of server (as used by client) + admin: + type: boolean + description: whether to enable the Admin API (default is false) + default: false mimetype: type: string description: default MIME type @@ -407,6 +411,7 @@ properties: editable: type: boolean description: whether the resource is editable + default: false table: type: string description: table name for RDBMS-based providers diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index ad49d7e..b48088b 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -50,14 +50,12 @@ from starlette.responses import ( import uvicorn from pygeoapi.api import API +from pygeoapi.admin import Admin from pygeoapi.openapi import load_openapi_document -from pygeoapi.util import yaml_load, get_api_rules +from pygeoapi.config import get_config +from pygeoapi.util import get_api_rules -if 'PYGEOAPI_CONFIG' not in os.environ: - raise RuntimeError('PYGEOAPI_CONFIG environment variable not set') - -with open(os.environ.get('PYGEOAPI_CONFIG'), encoding='utf8') as fh: - CONFIG = yaml_load(fh) +CONFIG = get_config() if 'PYGEOAPI_OPENAPI' not in os.environ: raise RuntimeError('PYGEOAPI_OPENAPI environment variable not set') @@ -479,6 +477,56 @@ async def stac_catalog_path(request: Request): return get_response(api_.get_stac_path(request, path)) +async def admin_config(request: Request): + """ + Admin endpoint + + :returns: Starlette HTTP Response + """ + + if request.method == 'GET': + return get_response(ADMIN.get_config(request)) + elif request.method == 'PUT': + return get_response(ADMIN.put_config(request)) + elif request.method == 'PATCH': + return get_response(ADMIN.patch_config(request)) + + +async def admin_config_resources(request: Request): + """ + Resources endpoint + + :returns: HTTP response + """ + + if request.method == 'GET': + return get_response(ADMIN.get_resources(request)) + elif request.method == 'POST': + return get_response(ADMIN.put_resource(request)) + + +async def admin_config_resource(request: Request, resource_id: str): + """ + Resource endpoint + + :param resource_id: resource identifier + + :returns: Starlette HTTP Response + """ + + if 'resource_id' in request.path_params: + resource_id = request.path_params['resource_id'] + + if request.method == 'GET': + return get_response(ADMIN.get_resource(request, resource_id)) + elif request.method == 'PUT': + return get_response(ADMIN.put_resource(request, resource_id)) + elif request.method == 'PATCH': + return get_response(ADMIN.patch_resource(request, resource_id)) + elif request.method == 'DELETE': + return get_response(ADMIN.delete_resource(request, resource_id)) + + class ApiRulesMiddleware: """ Custom middleware to properly deal with trailing slashes. See https://github.com/encode/starlette/issues/869. @@ -553,6 +601,17 @@ api_routes = [ Route('/stac/{path:path}', stac_catalog_path), ] +admin_routes = [ + Route('/admin/config', admin_config, methods=['GET', 'PUT', 'PATCH']), + Route('/admin/config/resources', admin_config_resources, methods=['GET', 'POST']), # noqa + Route('/admin/config/resources/{resource_id:path}', admin_config_resource, + methods=['GET', 'PUT', 'PATCH', 'DELETE']) +] + +if CONFIG['server'].get('admin', False): + ADMIN = Admin(CONFIG, OPENAPI) + api_routes.extend(admin_routes) + url_prefix = API_RULES.get_url_prefix('starlette') APP = Starlette( routes=[ diff --git a/pygeoapi/templates/_base.html b/pygeoapi/templates/_base.html index 1436fef..11e4898 100644 --- a/pygeoapi/templates/_base.html +++ b/pygeoapi/templates/_base.html @@ -46,6 +46,11 @@
  • Contact
    • + {% if config['server']['admin'] %} +
  • + {% trans %}Admin{% endtrans %} +
    • + {% endif %}