diff --git a/MANIFEST.in b/MANIFEST.in index 92505fd..ae0c43a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include README.md LICENSE.md requirements.txt -recursive-include pygeoapi *.html *.json +recursive-include pygeoapi *.html *.json *.yml recursive-include pygeoapi/static * diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a0c2109..f1c4fa5 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -60,7 +60,7 @@ function error() { cd ${PYGEOAPI_HOME} echo "Trying to generate openapi.yml" -pygeoapi openapi generate -c ${PYGEOAPI_CONFIG} > ${PYGEOAPI_OPENAPI} +pygeoapi openapi generate ${PYGEOAPI_CONFIG} > ${PYGEOAPI_OPENAPI} [[ $? -ne 0 ]] && error "openapi.yml could not be generated ERROR" diff --git a/docs/source/administration.rst b/docs/source/administration.rst index 5df5f52..ce00e03 100644 --- a/docs/source/administration.rst +++ b/docs/source/administration.rst @@ -20,19 +20,19 @@ To generate the OpenAPI document, run the following: .. code-block:: bash - pygeoapi openapi generate -c /path/to/my-pygeoapi-config.yml + pygeoapi openapi generate /path/to/my-pygeoapi-config.yml This will dump the OpenAPI document as YAML to your system's ``stdout``. To save to a file on disk, run: .. code-block:: bash - pygeoapi openapi generate -c /path/to/my-pygeoapi-config.yml > /path/to/my-pygeoapi-openapi.yml + pygeoapi openapi generate /path/to/my-pygeoapi-config.yml > /path/to/my-pygeoapi-openapi.yml To generate the OpenAPI document as JSON, run: .. code-block:: bash - pygeoapi openapi generate -c /path/to/my-pygeoapi-config.yml -f json > /path/to/my-pygeoapi-openapi.json + pygeoapi openapi generate /path/to/my-pygeoapi-config.yml -f json > /path/to/my-pygeoapi-openapi.json .. note:: Generate as YAML or JSON? If your OpenAPI YAML definition is slow to render as JSON, @@ -56,26 +56,7 @@ utility that can be run as follows: .. code-block:: bash - pygeoapi validate-openapi-document -o /path/to/my-pygeoapi-openapi.yml - - -Verifying configuration files ------------------------------ - -To ensure your YAML configurations are correctly formatted, you can use any YAML validator, or try -the Python one-liner per below: - -.. code-block:: bash - - python -c 'import yaml, sys; yaml.safe_load(sys.stdin)' < /path/to/my-pygeoapi-config.yml - python -c 'import yaml, sys; yaml.safe_load(sys.stdin)' < /path/to/my-pygeoapi-openapi.yml - -To ensure your OpenAPI JSON is correctly formatted, you can use any JSON validator, or try -the Python one-liner per below: - -.. code-block:: bash - - cat /path/to/my-pygeoapi-openapi.json | python -m json.tool + pygeoapi openapi validate /path/to/my-pygeoapi-openapi.yml Setting system environment variables diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index e7669c2..68a4928 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -200,6 +200,17 @@ default. :ref:`plugins` for more information on plugins +Validating the configuration +---------------------------- + +To ensure your configuration is valid, pygeoapi provides a validation +utility that can be run as follows: + +.. code-block:: bash + + pygeoapi config validate -c /path/to/my-pygeoapi-config.yml + + Using environment variables --------------------------- diff --git a/docs/source/data-publishing/ogcapi-records.rst b/docs/source/data-publishing/ogcapi-records.rst index 385998a..66211c9 100644 --- a/docs/source/data-publishing/ogcapi-records.rst +++ b/docs/source/data-publishing/ogcapi-records.rst @@ -3,7 +3,7 @@ Publishing metadata to OGC API - Records ======================================== -`OGC API - Records `_ provides geospatial data access functionality to vector data. +`OGC API - Records`_ provides geospatial data access functionality to vector data. To add vector data to pygeoapi, you can use the dataset example in :ref:`configuration` as a baseline and modify accordingly. diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 0cd54d0..d8c471a 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -31,7 +31,7 @@ For developers and the truly impatient vi example-config.yml export PYGEOAPI_CONFIG=example-config.yml export PYGEOAPI_OPENAPI=example-openapi.yml - pygeoapi openapi generate -c $PYGEOAPI_CONFIG > $PYGEOAPI_OPENAPI + pygeoapi openapi generate $PYGEOAPI_CONFIG > $PYGEOAPI_OPENAPI pygeoapi serve curl http://localhost:5000 diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml index a8f09ac..47d60de 100644 --- a/pygeoapi-config.yml +++ b/pygeoapi-config.yml @@ -165,7 +165,7 @@ resources: bbox: [-180,-90,180,90] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: - begin: 2011-11-11 + begin: 2011-11-11T11:11:11Z end: null # or empty (either means open ended) providers: - type: feature diff --git a/pygeoapi/__init__.py b/pygeoapi/__init__.py index ce3a65f..f4ddc6f 100644 --- a/pygeoapi/__init__.py +++ b/pygeoapi/__init__.py @@ -30,6 +30,7 @@ __version__ = '0.11.dev0' import click +from pygeoapi.config import config from pygeoapi.openapi import openapi @@ -58,4 +59,5 @@ def serve(ctx, server): raise click.ClickException('--flask/--starlette is required') +cli.add_command(config) cli.add_command(openapi) diff --git a/pygeoapi/config.py b/pygeoapi/config.py new file mode 100644 index 0000000..f9fe0d3 --- /dev/null +++ b/pygeoapi/config.py @@ -0,0 +1,83 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2021 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import click +import json +from jsonschema import validate as jsonschema_validate +import logging +import os + +from pygeoapi.util import to_json, yaml_load + +LOGGER = logging.getLogger(__name__) +THISDIR = os.path.dirname(os.path.realpath(__file__)) + + +def validate_config(instance_dict): + """ + Validate pygeoapi configuration against pygeoapi schema + + :param instance_dict: dict of configuration + + :returns: `bool` of validation + """ + + schema_file = os.path.join(THISDIR, 'schemas', 'config', + 'pygeoapi-config-0.x.yml') + + with open(schema_file) as fh2: + schema_dict = yaml_load(fh2) + jsonschema_validate(json.loads(to_json(instance_dict)), schema_dict) + + return True + + +@click.group() +def config(): + """Configuration management""" + pass + + +@click.command() +@click.pass_context +@click.option('--config', '-c', 'config_file', help='configuration file') +def validate(ctx, config_file): + """Validate configuration""" + + if config_file is None: + raise click.ClickException('--config/-c required') + + with open(config_file) as ff: + click.echo('Validating {}'.format(config_file)) + instance = yaml_load(ff) + validate_config(instance) + click.echo('Valid configuration') + + +config.add_command(validate) diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index f5578bf..08418ab 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -1070,7 +1070,7 @@ def openapi(): @click.command() @click.pass_context -@click.option('--config', '-c', 'config_file', help='configuration file') +@click.argument('config_file', type=click.File()) @click.option('--format', '-f', 'format_', type=click.Choice(['json', 'yaml']), default='yaml', help='output format (json|yaml)') def generate(ctx, config_file, format_='yaml'): @@ -1078,29 +1078,28 @@ def generate(ctx, config_file, format_='yaml'): if config_file is None: raise click.ClickException('--config/-c required') - with open(config_file) as ff: - s = yaml_load(ff) - pretty_print = s['server'].get('pretty_print', False) - if format_ == 'yaml': - click.echo(yaml.safe_dump(get_oas(s), default_flow_style=False)) - else: - click.echo(to_json(get_oas(s), pretty=pretty_print)) + + s = yaml_load(config_file) + pretty_print = s['server'].get('pretty_print', False) + if format_ == 'yaml': + click.echo(yaml.safe_dump(get_oas(s), default_flow_style=False)) + else: + click.echo(to_json(get_oas(s), pretty=pretty_print)) @click.command() @click.pass_context -@click.option('--openapi', '-o', 'openapi_file', help='OpenAPI document') +@click.argument('openapi_file', type=click.File()) def validate(ctx, openapi_file): """Validate OpenAPI Document""" if openapi_file is None: raise click.ClickException('--openapi/-o required') - with open(openapi_file) as ff: - click.echo('Validating {}'.format(openapi_file)) - instance = yaml_load(ff) - validate_openapi_document(instance) - click.echo('Valid OpenAPI document') + click.echo('Validating {}'.format(openapi_file)) + instance = yaml_load(openapi_file) + validate_openapi_document(instance) + click.echo('Valid OpenAPI document') openapi.add_command(generate) diff --git a/pygeoapi/schemas/config/pygeoapi-config-0.x.yml b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml new file mode 100644 index 0000000..090dbfc --- /dev/null +++ b/pygeoapi/schemas/config/pygeoapi-config-0.x.yml @@ -0,0 +1,469 @@ +$schema: https://json-schema.org/draft/2020-12/schema +$id: https://raw.githubusercontent.com/geopython/pygeoapi/master/pygeoapi/schemas/config/pygeoapi-config-0.x.yml +title: pygeoapi configuration schema +description: pygeoapi configuration schema + +type: object +properties: + server: + type: object + description: server object + properties: + bind: + type: object + description: binding server information + properties: + host: + type: string + description: binding IP + port: + type: integer + description: binding port + required: + - host + - port + url: + type: string + description: URL of server (as used by client) + mimetype: + type: string + description: default MIME type + encoding: + type: string + description: default server encoding + language: + type: string + description: default server language + cors: + type: boolean + description: boolean on whether server should support CORS + default: false + pretty_print: + type: boolean + description: whether JSON responses should be pretty-printed + default: false + limit: + type: integer + description: server limit on number of items to return + default: 10 + templates: + type: object + description: optional configuration to specify a different set of templates for HTML pages. Recommend using absolute paths. Omit this to use the default provided templates + properties: + path: + type: string + description: path to templates folder containing the jinja2 template HTML files + static: + type: string + description: path to static folder containing css, js, images and other static files referenced by the template + map: + type: object + description: leaflet map setup for HTML pages + properties: + url: + type: string + description: URI template of tile server + attribution: + type: string + description: map attribution + required: + - url + - attribution + ogc_schemas_location: + type: string + description: local copy of http://schemas.opengis.net + manager: + type: object + description: optional OGC API - Processes asynchronous job management + properties: + name: + type: string + description: plugin name (see `pygeoapi.plugin` for supported process_managers) + connection: + type: string + description: connection info to store jobs (e.g. filepath) + output_dir: + type: string + description: temporary file area for storing job results (files) + required: + - name + - connection + - output_dir + required: + - bind + - url + - mimetype + - encoding + - map + logging: + type: object + description: logging definitions + properties: + level: + type: string + description: |- + The logging level (see https://docs.python.org/3/library/logging.html#logging-levels). + If level is defined and logfile is undefined, logging messages are output to the server’s stdout + enum: + - CRITICAL + - ERROR + - WARNING + - INFO + - DEBUG + - NOTSET + logfile: + type: string + description: the full file path to the logfile. + required: + - level + metadata: + type: object + description: server metadata + properties: + identification: + type: object + description: server identification + properties: + title: + $ref: '#/definitions/i18n_string' + description: the title of the service + description: + $ref: '#/definitions/i18n_string' + description: some descriptive text about the service + keywords: + $ref: '#/definitions/i18n_array' + description: list of keywords about the service + keywords_type: + type: string + description: keyword type as per the ISO 19115 MD_KeywordTypeCode codelist + enum: + - discipline + - temporal + - place + - theme + - stratum + terms_of_service: + $ref: '#/definitions/i18n_string' + description: terms of service + url: + type: string + description: informative URL about the service + required: + - title + - description + - keywords + - url + license: + type: object + description: licensing details + properties: + name: + $ref: '#/definitions/i18n_string' + description: licensing details + url: + $ref: '#/definitions/i18n_string' + description: license URL + required: + - name + provider: + type: object + description: service provider details + properties: + name: + $ref: '#/definitions/i18n_string' + description: organization name + url: + $ref: '#/definitions/i18n_string' + description: URL of provider + required: + - name + contact: + type: object + description: service contact details + properties: + name: + type: string + description: Lastname, Firstname + position: + type: string + description: position + address: + type: string + description: postal address + city: + type: string + description: city + stateorprovince: + type: string + description: administrative area + postalcode: + type: string + description: postal or ZIP code + country: + type: string + description: country + phone: + type: string + description: phone number + fax: + type: string + description: fax number + email: + type: string + description: email address + url: + type: string + description: URL of contact + hours: + type: string + description: hours of service + instructions: + type: string + description: contact instructions + role: + type: string + description: role as per the ISO 19115 CI_RoleCode codelist + required: + - name + required: + - identification + - license + - provider + - contact + resources: + type: object + description: collections or processes published by the server + patternProperties: + "^.*$": + anyOf: + - type: object + description: base resource object + properties: + type: + type: string + description: resource type + enum: + - collection + - stac-collection + title: + $ref: '#/definitions/i18n_string' + description: the title of the service + description: + $ref: '#/definitions/i18n_string' + description: some descriptive text about the service + keywords: + $ref: '#/definitions/i18n_array' + description: list of keywords about the service + context: + type: array + description: linked data configuration + items: + - type: object + patternProperties: + "^.*$": + anyOf: + - type: string + - type: object + links: + type: array + description: list of related links + minItems: 0 + items: + type: object + properties: + type: + type: string + description: MIME type + rel: + type: string + description: link relations per https://www.iana.org/assignments/link-relations/link-relations.xhtml + title: + type: string + description: title + href: + type: string + description: URL + hreflang: + type: string + description: language + required: + - type + - rel + - href + extents: + type: object + description: spatial and temporal extents + properties: + spatial: + type: object + description: spatial extent and CRS + properties: + bbox: + type: array + description: bounding box of resource + items: + type: number + minItems: 4 + maxItems: 6 + crs: + type: string + description: coordinate reference system of bbox + default: 'http://www.opengis.net/def/crs/OGC/1.3/CRS84' + required: + - bbox + temporal: + type: object + description: temporal extent of resource + properties: + begin: + type: [string, 'null'] + format: date-time + nullable: true + end: + type: [string, 'null'] + format: date-time + nullable: true + required: + - spatial + providers: + type: array + description: required connection information + items: + type: object + properties: + type: + type: string + description: underlying data geospatial type + enum: + - feature + - coverage + - record + - tile + - edr + - stac + default: + type: boolean + description: |- + whether the provider is the default. If not specified, the + first provider definition is considered the default + name: + type: string + description: |- + see `pygeoapi.plugin` for supported provider names. + For custom built plugins, use the import path (e.g. `mypackage.provider.MyProvider`) + data: + anyOf: + - type: string + - type: object + description: the data filesystem path or URL, depending on plugin setup + table: + type: string + description: table name for RDBMS-based providers + id_field: + type: string + description: required for vector data, the field corresponding to the ID + geometry: + type: object + description: the field corresponding to the geometry + properties: + x_field: + type: string + description: the field corresponding to the x geometry + y_field: + type: string + description: the field corresponding to the y geometry + required: + - x_field + - y_field + time_field: + type: string + description: optional field corresponding to the temporal property of the dataset + title_field: + type: string + description: optional field of which property to display as title/label on HTML pages + format: + type: object + description: default format + properties: + name: + type: string + description: format name + mimetype: + type: string + description: format mimetype + required: + - name + - mimetype + options: + type: object + description: optional options key value pairs to pass to provider (i.e. GDAL creation) + patternProperties: + "^[a-z]{2}$": + allOf: + - type: string + properties: + type: array + description: only return the following properties, in order + items: + type: string + minItems: 1 + uniqueItems: true + required: + - type + - name + - data + required: + - type + - title + - description + - keywords + - extents + - providers + - type: object + description: process object + properties: + type: + type: string + description: resource type + enum: + - process + processor: + type: object + description: process binding + properties: + name: + type: string + description: |- + see `pygeoapi.plugin` for supported provider names. + For custom built plugins, use the import path (e.g. `mypackage.provider.MyProvider`) + required: + - name + required: + - type + - processor +definitions: + i18n_string: + oneOf: + - type: string + - type: object + patternProperties: + "^[a-zA-Z]{2,3}([-_][a-zA-Z0-9]{2,3})?$": + allOf: + - type: string + i18n_array: + oneOf: + - type: array + items: + type: string + - type: object + patternProperties: + "^[a-zA-Z]{2,3}([-_][a-zA-Z0-9]{2,3})?$": + allOf: + - type: array + items: + type: string +required: + - server + - logging + - metadata + - resources diff --git a/pygeoapi/util.py b/pygeoapi/util.py index e99277b..cfb914b 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -456,7 +456,7 @@ def get_provider_default(providers): try: default = (next(d for i, d in enumerate(providers) if 'default' in d - and d['default'] is True)) + and d['default'])) LOGGER.debug('found default provider type') except StopIteration: LOGGER.debug('no default provider type. Returning first provider') diff --git a/tests/cite/ogcapi-features/README.md b/tests/cite/ogcapi-features/README.md index cdb55bd..a0bf4c3 100644 --- a/tests/cite/ogcapi-features/README.md +++ b/tests/cite/ogcapi-features/README.md @@ -14,6 +14,6 @@ pip install gunicorn cd tests/cite/ogcapi-features . cite.env python ../../load_es_data.py ./canada-hydat-daily-mean-02hc003.geojson IDENTIFIER -pygeoapi openapi generate -c $PYGEOAPI_CONFIG > $PYGEOAPI_OPENAPI +pygeoapi openapi generate $PYGEOAPI_CONFIG > $PYGEOAPI_OPENAPI gunicorn pygeoapi.flask_app:APP -b 0.0.0.0:5001 --access-logfile '-' ``` diff --git a/tests/pygeoapi-test-config.yml b/tests/pygeoapi-test-config.yml index 00f1bee..753a759 100644 --- a/tests/pygeoapi-test-config.yml +++ b/tests/pygeoapi-test-config.yml @@ -48,6 +48,7 @@ server: manager: name: TinyDB connection: /tmp/pygeoapi-test-process-manager.db + output_dir: /tmp logging: level: ERROR @@ -195,7 +196,7 @@ resources: bbox: [-180,-90,180,90] crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 temporal: - begin: 2011-11-11 + begin: 2011-11-11T11:11:11Z end: null # or empty (either means open ended) providers: - type: feature diff --git a/tests/pygeoapi-test-ogr-config.yml b/tests/pygeoapi-test-ogr-config.yml index 768be7f..30f5c61 100644 --- a/tests/pygeoapi-test-ogr-config.yml +++ b/tests/pygeoapi-test-ogr-config.yml @@ -360,6 +360,7 @@ resources: id_field: objectid cases_italy_per_region_from_github: + type: collection title: "Cases in Italy - DPC GitHub" description: "Current situation within Italy, number of cases with variation per Italy, provided by ESRI, source data from DPC." keywords: [Daily, Cases Variation, Region] diff --git a/tests/test_config.py b/tests/test_config.py index 73dc49c..623afb2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -29,13 +29,21 @@ import os +from jsonschema.exceptions import ValidationError import pytest +from pygeoapi.config import validate_config from pygeoapi.util import yaml_load from .util import get_test_file_path +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + return yaml_load(fh) + + def test_config_envvars(): os.environ['PYGEOAPI_PORT'] = '5001' os.environ['PYGEOAPI_TITLE'] = 'my title' @@ -53,3 +61,11 @@ def test_config_envvars(): with pytest.raises(EnvironmentError): with open(get_test_file_path('pygeoapi-test-config-envvars.yml')) as fh: # noqa config = yaml_load(fh) + + +def test_validate_config(config): + is_valid = validate_config(config) + assert is_valid + + with pytest.raises(ValidationError): + is_valid = validate_config({'foo': 'bar'}) diff --git a/tests/test_openapi.py b/tests/test_openapi.py index 0c210c6..24e7560 100644 --- a/tests/test_openapi.py +++ b/tests/test_openapi.py @@ -29,6 +29,8 @@ import pytest +from jsonschema.exceptions import ValidationError + from pygeoapi.openapi import (get_oas, get_ogc_schemas_location, validate_openapi_document) from pygeoapi.util import yaml_load @@ -72,10 +74,12 @@ def test_get_oas(config, openapi): is_valid = validate_openapi_document(openapi_doc) - assert is_valid is True + assert is_valid def test_validate_openapi_document(openapi): is_valid = validate_openapi_document(openapi) + assert is_valid - assert is_valid is True + with pytest.raises(ValidationError): + is_valid = validate_openapi_document({'foo': 'bar'}) diff --git a/tests/test_util.py b/tests/test_util.py index e7e0d3f..88ea5d1 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -59,19 +59,19 @@ def test_yaml_load(): def test_str2bool(): - assert util.str2bool(False) is False - assert util.str2bool('0') is False - assert util.str2bool('no') is False - assert util.str2bool('yes') is True - assert util.str2bool('1') is True - assert util.str2bool(True) is True - assert util.str2bool('true') is True - assert util.str2bool('True') is True - assert util.str2bool('TRUE') is True - assert util.str2bool('tRuE') is True - assert util.str2bool('on') is True - assert util.str2bool('On') is True - assert util.str2bool('off') is False + assert not util.str2bool(False) + assert not util.str2bool('0') + assert not util.str2bool('no') + assert util.str2bool('yes') + assert util.str2bool('1') + assert util.str2bool(True) + assert util.str2bool('true') + assert util.str2bool('True') + assert util.str2bool('TRUE') + assert util.str2bool('tRuE') + assert util.str2bool('on') + assert util.str2bool('On') + assert not util.str2bool('off') def test_json_serial():