From 2328e1d66f52aeda0690a8c00b1ff8d4f456f1c8 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 20 Oct 2019 22:36:37 -0400 Subject: [PATCH 1/2] implement env variables inside PYGEOAPI_CONFIG file (#248) --- pygeoapi/util.py | 49 ++++++++-- tests/pygeoapi-test-config-envvars.yml | 120 +++++++++++++++++++++++++ tests/test_config.py | 59 ++++++++++++ tests/test_util.py | 102 +++++++++++++++++++++ 4 files changed, 322 insertions(+), 8 deletions(-) create mode 100644 tests/pygeoapi-test-config-envvars.yml create mode 100644 tests/test_config.py create mode 100644 tests/test_util.py diff --git a/pygeoapi/util.py b/pygeoapi/util.py index fe37953..e65affb 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2019 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -32,12 +32,36 @@ from datetime import date, datetime, time from decimal import Decimal import logging +import os +import re import yaml LOGGER = logging.getLogger(__name__) +def get_typed_value(value): + """ + Derive true type from data value + + :param value: value + + :returns: value as a native Python data type + """ + + try: + if '.' in value: # float? + value2 = float(value) + elif len(value) > 1 and value.startswith('0'): + value2 = value + else: # int? + value2 = int(value) + except ValueError: # string (default)? + value2 = value + + return value2 + + def get_url(scheme, host, port, basepath): """ Provides URL of instance @@ -64,11 +88,21 @@ def yaml_load(fh): :returns: `dict` representation of YAML """ - try: - return yaml.load(fh, Loader=yaml.FullLoader) - except AttributeError as err: - LOGGER.warning('YAML loading error: {}'.format(err)) - return yaml.load(fh) + # support environment variables in config + # https://stackoverflow.com/a/55301129 + # + path_matcher = re.compile(r'.*\$\{([^}^{]+)\}.*') + + def path_constructor(loader, node): + return get_typed_value(os.path.expandvars(node.value)) + + class EnvVarLoader(yaml.SafeLoader): + pass + + EnvVarLoader.add_implicit_resolver('!path', path_matcher, None) + EnvVarLoader.add_constructor('!path', path_constructor) + + return yaml.load(fh, Loader=EnvVarLoader) def str2bool(value): @@ -100,8 +134,7 @@ def json_serial(obj): """ if isinstance(obj, (datetime, date, time)): - serial = obj.isoformat() - return serial + return obj.isoformat() elif isinstance(obj, Decimal): return float(obj) diff --git a/tests/pygeoapi-test-config-envvars.yml b/tests/pygeoapi-test-config-envvars.yml new file mode 100644 index 0000000..717f94f --- /dev/null +++ b/tests/pygeoapi-test-config-envvars.yml @@ -0,0 +1,120 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2019 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +server: + bind: + host: 0.0.0.0 + port: ${PYGEOAPI_PORT} + url: http://localhost:5000/ + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + language: en-US + cors: true + pretty_print: true + limit: 10 + # templates: /path/to/templates + map: + url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png + attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + +logging: + level: ERROR + #logfile: /tmp/pygeoapi.log + +metadata: + identification: + title: pygeoapi default instance ${PYGEOAPI_TITLE} + description: pygeoapi provides an API to geospatial data + keywords: + - geospatial + - data + - api + keywords_type: theme + terms_of_service: None + url: http://example.org + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: Organization Name + url: https://pygeoapi.io + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Hours of Service + instructions: During hours of service. Off on weekends. + role: pointOfContact + +datasets: + obs: + title: Observations + description: My cool observations + keywords: + - observations + - monitoring + links: + - type: text/csv + rel: canonical + title: data + href: https://github.com/mapserver/mapserver/blob/branch-7-0/msautotest/wxs/data/obs.csv + hreflang: en-US + - type: text/csv + rel: alternate + title: data + href: https://raw.githubusercontent.com/mapserver/mapserver/branch-7-0/msautotest/wxs/data/obs.csv + hreflang: en-US + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2000-10-30T18:24:39Z + end: 2007-10-30T08:57:29Z + trs: http://www.opengis.net/def/uom/ISO-8601/0/Gregorian + provider: + name: CSV + data: tests/data/obs.csv + id_field: id + geometry: + x_field: long + y_field: lat + +processes: + hello-world: + processor: + name: HelloWorld diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..b405eb4 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,59 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2019 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import os + +import pytest + +from pygeoapi.util import yaml_load + +os.environ['PYGEOAPI_PORT'] = '5001' +os.environ['PYGEOAPI_TITLE'] = 'my title' + + +def get_test_file_path(filename): + """helper function to open test file safely""" + + if os.path.isfile(filename): + return filename + else: + return 'tests/{}'.format(filename) + + +@pytest.fixture() +def config(): + with open(get_test_file_path('pygeoapi-test-config-envvars.yml')) as fh: + return yaml_load(fh) + + +def test_config_envvars(config): + assert isinstance(config, dict) + assert config['server']['bind']['port'] == 5001 + assert config['metadata']['identification']['title'] == \ + 'pygeoapi default instance my title' diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..e5f8f95 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,102 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2019 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +from datetime import datetime, date, time +from decimal import Decimal +import os + +import pytest + +from pygeoapi import util + + +def get_test_file_path(filename): + """helper function to open test file safely""" + + if os.path.isfile(filename): + return filename + else: + return 'tests/{}'.format(filename) + + +def test_get_typed_value(): + value = util.get_typed_value('2') + assert isinstance(value, int) + + value = util.get_typed_value('1.2') + assert isinstance(value, float) + + value = util.get_typed_value('1.c2') + assert isinstance(value, str) + + +def test_get_url(): + url = util.get_url('http', 'example.org', 8002, '/b') + assert url == 'http://example.org:8002/b' + + url = util.get_url('http', 'example.org', 80, '/b') + assert url == 'http://example.org/b' + + +def test_yaml_load(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + d = util.yaml_load(fh) + assert isinstance(d, dict) + with pytest.raises(FileNotFoundError): + with open(get_test_file_path('ipygeoapi-test-config.yml')) as fh: + d = util.yaml_load(fh) + + +def test_str2bool(): + assert util.str2bool(False) is False + assert util.str2bool('0') is False + assert util.str2bool('no') is False + assert util.str2bool('yes') is True + assert util.str2bool('1') is True + assert util.str2bool(True) is True + assert util.str2bool('true') is True + assert util.str2bool('on') is True + assert util.str2bool('off') is False + + +def test_json_serial(): + d = datetime(1972, 10, 30) + assert util.json_serial(d) == '1972-10-30T00:00:00' + + d = date(2010, 7, 31) + assert util.json_serial(d) == '2010-07-31' + + d = time(11) + assert util.json_serial(d) == '11:00:00' + + d = Decimal(1.0) + assert util.json_serial(d) == 1.0 + + with pytest.raises(TypeError): + util.json_serial('foo') From efc563593bd5d6688654d5b06e21430304874ca7 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 21 Oct 2019 09:53:16 -0400 Subject: [PATCH 2/2] add error handler for not defined environment variables --- pygeoapi/util.py | 21 +++------------------ tests/test_config.py | 19 +++++++++++-------- tests/test_util.py | 14 +++++--------- 3 files changed, 19 insertions(+), 35 deletions(-) diff --git a/pygeoapi/util.py b/pygeoapi/util.py index e65affb..7d8b331 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -62,23 +62,6 @@ def get_typed_value(value): return value2 -def get_url(scheme, host, port, basepath): - """ - Provides URL of instance - - :returns: string of complete baseurl - """ - - url = '{}://{}'.format(scheme, host) - - if port not in [80, 443]: - url = '{}:{}'.format(url, port) - - url = '{}{}'.format(url, basepath) - - return url - - def yaml_load(fh): """ serializes a YAML files into a pyyaml object @@ -90,10 +73,12 @@ def yaml_load(fh): # support environment variables in config # https://stackoverflow.com/a/55301129 - # path_matcher = re.compile(r'.*\$\{([^}^{]+)\}.*') def path_constructor(loader, node): + env_var = path_matcher.match(node.value).group(1) + if env_var not in os.environ: + raise EnvironmentError('Undefined environment variable in config') return get_typed_value(os.path.expandvars(node.value)) class EnvVarLoader(yaml.SafeLoader): diff --git a/tests/test_config.py b/tests/test_config.py index b405eb4..7d2b455 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -33,9 +33,6 @@ import pytest from pygeoapi.util import yaml_load -os.environ['PYGEOAPI_PORT'] = '5001' -os.environ['PYGEOAPI_TITLE'] = 'my title' - def get_test_file_path(filename): """helper function to open test file safely""" @@ -46,14 +43,20 @@ def get_test_file_path(filename): return 'tests/{}'.format(filename) -@pytest.fixture() -def config(): +def test_config_envvars(): + os.environ['PYGEOAPI_PORT'] = '5001' + os.environ['PYGEOAPI_TITLE'] = 'my title' + with open(get_test_file_path('pygeoapi-test-config-envvars.yml')) as fh: - return yaml_load(fh) + config = yaml_load(fh) - -def test_config_envvars(config): assert isinstance(config, dict) assert config['server']['bind']['port'] == 5001 assert config['metadata']['identification']['title'] == \ 'pygeoapi default instance my title' + + os.environ.pop('PYGEOAPI_PORT') + + with pytest.raises(EnvironmentError): + with open(get_test_file_path('pygeoapi-test-config-envvars.yml')) as fh: # noqa + config = yaml_load(fh) diff --git a/tests/test_util.py b/tests/test_util.py index e5f8f95..f4b8aca 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -56,20 +56,12 @@ def test_get_typed_value(): assert isinstance(value, str) -def test_get_url(): - url = util.get_url('http', 'example.org', 8002, '/b') - assert url == 'http://example.org:8002/b' - - url = util.get_url('http', 'example.org', 80, '/b') - assert url == 'http://example.org/b' - - def test_yaml_load(): with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: d = util.yaml_load(fh) assert isinstance(d, dict) with pytest.raises(FileNotFoundError): - with open(get_test_file_path('ipygeoapi-test-config.yml')) as fh: + with open(get_test_file_path('404.yml')) as fh: d = util.yaml_load(fh) @@ -81,7 +73,11 @@ def test_str2bool(): assert util.str2bool('1') is True assert util.str2bool(True) is True assert util.str2bool('true') is True + assert util.str2bool('True') is True + assert util.str2bool('TRUE') is True + assert util.str2bool('tRuE') is True assert util.str2bool('on') is True + assert util.str2bool('On') is True assert util.str2bool('off') is False