Merge pull request #282 from geopython/issue-248-env

implement env variables inside PYGEOAPI_CONFIG file (#248)
2019-10-22 09:26:29 +02:00
parent c8d0f1b512 efc563593b
commit 3bd0d280cc
4 changed files with 316 additions and 18 deletions
@@ -2,7 +2,7 @@
 #
 # Authors: Tom Kralidis <tomkralidis@gmail.com>
 #
-# Copyright (c) 2018 Tom Kralidis
+# Copyright (c) 2019 Tom Kralidis
 #
 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation
@@ -32,27 +32,34 @@
 from datetime import date, datetime, time
 from decimal import Decimal
 import logging
+import os
+import re

 import yaml

 LOGGER = logging.getLogger(__name__)


-def get_url(scheme, host, port, basepath):
+def get_typed_value(value):
    """
-    Provides URL of instance
+    Derive true type from data value

-    :returns: string of complete baseurl
+    :param value: value
+
+    :returns: value as a native Python data type
    """

-    url = '{}://{}'.format(scheme, host)
+    try:
+        if '.' in value:  # float?
+            value2 = float(value)
+        elif len(value) > 1 and value.startswith('0'):
+            value2 = value
+        else:  # int?
+            value2 = int(value)
+    except ValueError:  # string (default)?
+        value2 = value

-    if port not in [80, 443]:
-        url = '{}:{}'.format(url, port)
-
-    url = '{}{}'.format(url, basepath)
-
-    return url
+    return value2


 def yaml_load(fh):
@@ -64,11 +71,23 @@ def yaml_load(fh):
    :returns: `dict` representation of YAML
    """

-    try:
-        return yaml.load(fh, Loader=yaml.FullLoader)
-    except AttributeError as err:
-        LOGGER.warning('YAML loading error: {}'.format(err))
-        return yaml.load(fh)
+    # support environment variables in config
+    # https://stackoverflow.com/a/55301129
+    path_matcher = re.compile(r'.*\$\{([^}^{]+)\}.*')
+
+    def path_constructor(loader, node):
+        env_var = path_matcher.match(node.value).group(1)
+        if env_var not in os.environ:
+            raise EnvironmentError('Undefined environment variable in config')
+        return get_typed_value(os.path.expandvars(node.value))
+
+    class EnvVarLoader(yaml.SafeLoader):
+        pass
+
+    EnvVarLoader.add_implicit_resolver('!path', path_matcher, None)
+    EnvVarLoader.add_constructor('!path', path_constructor)
+
+    return yaml.load(fh, Loader=EnvVarLoader)


 def str2bool(value):
@@ -100,8 +119,7 @@ def json_serial(obj):
    """

    if isinstance(obj, (datetime, date, time)):
-        serial = obj.isoformat()
-        return serial
+        return obj.isoformat()
    elif isinstance(obj, Decimal):
        return float(obj)

@@ -0,0 +1,120 @@
+# =================================================================
+#
+# Authors: Tom Kralidis <tomkralidis@gmail.com>
+#
+# Copyright (c) 2019 Tom Kralidis
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+
+server:
+    bind:
+        host: 0.0.0.0
+        port: ${PYGEOAPI_PORT}
+    url: http://localhost:5000/
+    mimetype: application/json; charset=UTF-8
+    encoding: utf-8
+    language: en-US
+    cors: true
+    pretty_print: true
+    limit: 10
+    # templates: /path/to/templates
+    map:
+        url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png
+        attribution: '<a href="https://wikimediafoundation.org/wiki/Maps_Terms_of_Use">Wikimedia maps</a> | Map data &copy; <a href="https://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
+
+logging:
+    level: ERROR
+    #logfile: /tmp/pygeoapi.log
+
+metadata:
+    identification:
+        title: pygeoapi default instance ${PYGEOAPI_TITLE}
+        description: pygeoapi provides an API to geospatial data
+        keywords:
+            - geospatial
+            - data
+            - api
+        keywords_type: theme
+        terms_of_service: None
+        url: http://example.org
+    license:
+        name: CC-BY 4.0 license
+        url: https://creativecommons.org/licenses/by/4.0/
+    provider:
+        name: Organization Name
+        url: https://pygeoapi.io
+    contact:
+        name: Lastname, Firstname
+        position: Position Title
+        address: Mailing Address
+        city: City
+        stateorprovince: Administrative Area
+        postalcode: Zip or Postal Code
+        country: Country
+        phone: +xx-xxx-xxx-xxxx
+        fax: +xx-xxx-xxx-xxxx
+        email: you@example.org
+        url: Contact URL
+        hours: Hours of Service
+        instructions: During hours of service.  Off on weekends.
+        role: pointOfContact
+
+datasets:
+    obs:
+        title: Observations
+        description: My cool observations
+        keywords:
+            - observations
+            - monitoring
+        links:
+            - type: text/csv
+              rel: canonical
+              title: data
+              href: https://github.com/mapserver/mapserver/blob/branch-7-0/msautotest/wxs/data/obs.csv
+              hreflang: en-US
+            - type: text/csv
+              rel: alternate
+              title: data
+              href: https://raw.githubusercontent.com/mapserver/mapserver/branch-7-0/msautotest/wxs/data/obs.csv
+              hreflang: en-US
+        extents:
+            spatial:
+                bbox: [-180,-90,180,90]
+                crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
+            temporal:
+                begin: 2000-10-30T18:24:39Z
+                end: 2007-10-30T08:57:29Z
+                trs: http://www.opengis.net/def/uom/ISO-8601/0/Gregorian
+        provider:
+            name: CSV
+            data: tests/data/obs.csv
+            id_field: id
+            geometry:
+                x_field: long
+                y_field: lat
+
+processes:
+    hello-world:
+        processor:
+            name: HelloWorld
@@ -0,0 +1,62 @@
+# =================================================================
+#
+# Authors: Tom Kralidis <tomkralidis@gmail.com>
+#
+# Copyright (c) 2019 Tom Kralidis
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+
+import os
+
+import pytest
+
+from pygeoapi.util import yaml_load
+
+
+def get_test_file_path(filename):
+    """helper function to open test file safely"""
+
+    if os.path.isfile(filename):
+        return filename
+    else:
+        return 'tests/{}'.format(filename)
+
+
+def test_config_envvars():
+    os.environ['PYGEOAPI_PORT'] = '5001'
+    os.environ['PYGEOAPI_TITLE'] = 'my title'
+
+    with open(get_test_file_path('pygeoapi-test-config-envvars.yml')) as fh:
+        config = yaml_load(fh)
+
+    assert isinstance(config, dict)
+    assert config['server']['bind']['port'] == 5001
+    assert config['metadata']['identification']['title'] == \
+        'pygeoapi default instance my title'
+
+    os.environ.pop('PYGEOAPI_PORT')
+
+    with pytest.raises(EnvironmentError):
+        with open(get_test_file_path('pygeoapi-test-config-envvars.yml')) as fh:  # noqa
+            config = yaml_load(fh)
@@ -0,0 +1,98 @@
+# =================================================================
+#
+# Authors: Tom Kralidis <tomkralidis@gmail.com>
+#
+# Copyright (c) 2019 Tom Kralidis
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+
+from datetime import datetime, date, time
+from decimal import Decimal
+import os
+
+import pytest
+
+from pygeoapi import util
+
+
+def get_test_file_path(filename):
+    """helper function to open test file safely"""
+
+    if os.path.isfile(filename):
+        return filename
+    else:
+        return 'tests/{}'.format(filename)
+
+
+def test_get_typed_value():
+    value = util.get_typed_value('2')
+    assert isinstance(value, int)
+
+    value = util.get_typed_value('1.2')
+    assert isinstance(value, float)
+
+    value = util.get_typed_value('1.c2')
+    assert isinstance(value, str)
+
+
+def test_yaml_load():
+    with open(get_test_file_path('pygeoapi-test-config.yml')) as fh:
+        d = util.yaml_load(fh)
+        assert isinstance(d, dict)
+    with pytest.raises(FileNotFoundError):
+        with open(get_test_file_path('404.yml')) as fh:
+            d = util.yaml_load(fh)
+
+
+def test_str2bool():
+    assert util.str2bool(False) is False
+    assert util.str2bool('0') is False
+    assert util.str2bool('no') is False
+    assert util.str2bool('yes') is True
+    assert util.str2bool('1') is True
+    assert util.str2bool(True) is True
+    assert util.str2bool('true') is True
+    assert util.str2bool('True') is True
+    assert util.str2bool('TRUE') is True
+    assert util.str2bool('tRuE') is True
+    assert util.str2bool('on') is True
+    assert util.str2bool('On') is True
+    assert util.str2bool('off') is False
+
+
+def test_json_serial():
+    d = datetime(1972, 10, 30)
+    assert util.json_serial(d) == '1972-10-30T00:00:00'
+
+    d = date(2010, 7, 31)
+    assert util.json_serial(d) == '2010-07-31'
+
+    d = time(11)
+    assert util.json_serial(d) == '11:00:00'
+
+    d = Decimal(1.0)
+    assert util.json_serial(d) == 1.0
+
+    with pytest.raises(TypeError):
+        util.json_serial('foo')