diff --git a/README.md b/README.md index ea524ce..2155aae 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ curl http://localhost:5000/collections/countries/items?resulttype=hits docker pull swaggerapi/swagger-ui docker run -p 80:8080 swaggerapi/swagger-ui # go to http://localhost -# enter http://localhost:5000/api and click 'Explore' +# enter http://localhost:5000/openapi and click 'Explore' ``` ## Demo Server diff --git a/docs/source/openapi.rst b/docs/source/openapi.rst index 59259fb..f8af172 100644 --- a/docs/source/openapi.rst +++ b/docs/source/openapi.rst @@ -15,9 +15,9 @@ pygeoapi REST end points descriptions on OpenAPI standard are automatically gene pygeoapi generate-openapi-document -c local.config.yml > openapi.yml -The api will them be accessible at `/api` endpoint. +The api will them be accessible at `/openapi` endpoint. -For api demo please check: ``_ +For api demo please check: ``_ The api page has REST description but also integrated clients that can be used to send requests to the REST end points and see the response provided diff --git a/pygeoapi/api.py b/pygeoapi/api.py index 250e21e..93bdcea 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -146,12 +146,12 @@ class API(object): 'rel': 'service-desc', 'type': 'application/vnd.oai.openapi+json;version=3.0', 'title': 'The OpenAPI definition as JSON', - 'href': '{}/api'.format(self.config['server']['url']) + 'href': '{}/openapi'.format(self.config['server']['url']) }, { 'rel': 'service-doc', 'type': 'text/html', 'title': 'The OpenAPI definition as HTML', - 'href': '{}/api?f=html'.format(self.config['server']['url']), + 'href': '{}/openapi?f=html'.format(self.config['server']['url']), 'hreflang': self.config['server']['language'] }, { 'rel': 'conformance', @@ -178,7 +178,7 @@ class API(object): return headers_, 200, json.dumps(fcm) @pre_process - def api(self, headers_, format_, openapi): + def openapi(self, headers_, format_, openapi): """ Provide OpenAPI document @@ -199,14 +199,14 @@ class API(object): LOGGER.error(exception) return headers_, 400, json.dumps(exception) - path = '/'.join([self.config['server']['url'].rstrip('/'), 'api']) + path = '/'.join([self.config['server']['url'].rstrip('/'), 'openapi']) if format_ == 'html': data = { 'openapi-document-path': path } headers_['Content-Type'] = 'text/html' - content = _render_j2_template(self.config, 'api.html', data) + content = _render_j2_template(self.config, 'openapi.html', data) return headers_, 200, content headers_['Content-Type'] = \ diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index 2053069..09e343d 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -77,8 +77,8 @@ def root(): return response -@APP.route('/api') -def api(): +@APP.route('/openapi') +def openapi(): """ OpenAPI access point @@ -87,8 +87,8 @@ def api(): with open(os.environ.get('PYGEOAPI_OPENAPI')) as ff: openapi = yaml_load(ff) - headers, status_code, content = api_.api(request.headers, request.args, - openapi) + headers, status_code, content = api_.openapi(request.headers, request.args, + openapi) response = make_response(content, status_code) if headers: diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index d95a2e2..b8fbc44 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -146,7 +146,7 @@ def get_oas_30(cfg): } } - paths['/api'] = { + paths['/openapi'] = { 'get': { 'summary': 'This document', 'description': 'This document', diff --git a/pygeoapi/provider/postgresql.py b/pygeoapi/provider/postgresql.py index cfab560..db1b146 100644 --- a/pygeoapi/provider/postgresql.py +++ b/pygeoapi/provider/postgresql.py @@ -72,8 +72,11 @@ class DatabaseConnection(object): (defaults to UNIX socket if not provided) port – connection port number (defaults to 5432 if not provided) - schema – schema to use as search path, normally - data is in the public schema + search_path – search path to be used (by order) , normally + data is in the public schema, [public], + or in a specific schema ["osm", "public"]. + Note: First we should have the schema + being used and then public :param table: table name containing the data. This variable is used to assemble column information @@ -87,18 +90,14 @@ class DatabaseConnection(object): self.context = context self.columns = None self.conn = None - self.schema = None def __enter__(self): try: - self.schema = self.conn_dic.pop('schema', None) - if self.schema == 'public' or self.schema is None: - pass - else: - self.conn_dic["options"] = '-c search_path={}'.format( - self.schema) - LOGGER.debug('Using schema {} as search path'.format( - self.schema)) + search_path = self.conn_dic.pop('search_path', ['public']) + if search_path != ['public']: + self.conn_dic["options"] = f'-c \ + search_path={",".join(search_path)}' + LOGGER.debug(f'Using search path: {search_path} ') self.conn = psycopg2.connect(**self.conn_dic) except psycopg2.OperationalError: @@ -149,6 +148,7 @@ class PostgreSQLProvider(BaseProvider): self.table = provider_def['table'] self.id_field = provider_def['id_field'] self.conn_dic = provider_def['data'] + self.geom = provider_def.get('geom_field', 'geom') LOGGER.debug('Setting Postgresql properties:') LOGGER.debug('Connection String:{}'.format( @@ -188,7 +188,6 @@ class PostgreSQLProvider(BaseProvider): except Exception as err: LOGGER.error('Error executing sql_query: {}: {}'.format( sql_query.as_string(cursor)), err) - LOGGER.error('Using public schema: {}'.format(db.schema)) raise ProviderQueryError() hits = cursor.fetchone()["hits"] @@ -203,9 +202,9 @@ class PostgreSQLProvider(BaseProvider): SELECT {},ST_AsGeoJSON({}) FROM {} WHERE {} @ \ ST_MakeEnvelope({}, {}, {}, {})").\ format(db.columns, - Identifier('geom'), + Identifier(self.geom), Identifier(self.table), - Identifier('geom'), + Identifier(self.geom), Placeholder(), Placeholder(), Placeholder(), @@ -214,7 +213,7 @@ class PostgreSQLProvider(BaseProvider): if not bbox: bbox = [-180, -90, 180, 90] - LOGGER.debug('SQL Query: {}'.format(sql_query.as_string(cursor))) + LOGGER.debug('SQL Query: {}'.format(sql_query)) LOGGER.debug('Start Index: {}'.format(startindex)) LOGGER.debug('End Index: {}'.format(end_index)) try: @@ -225,7 +224,6 @@ class PostgreSQLProvider(BaseProvider): except Exception as err: LOGGER.error('Error executing sql_query: {}'.format( sql_query.as_string(cursor))) - LOGGER.error('Using public schema: {}'.format(db.schema)) LOGGER.error(err) raise ProviderQueryError() @@ -258,7 +256,7 @@ class PostgreSQLProvider(BaseProvider): sql_query = SQL("select {},ST_AsGeoJSON({}) \ from {} WHERE {}=%s").format(db.columns, - Identifier('geom'), + Identifier(self.geom), Identifier(self.table), Identifier(self.id_field)) @@ -269,7 +267,6 @@ class PostgreSQLProvider(BaseProvider): except Exception as err: LOGGER.error('Error executing sql_query: {}'.format( sql_query.as_string(cursor))) - LOGGER.error('Using public schema: {}'.format(db.schema)) LOGGER.error(err) raise ProviderQueryError() diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 29f3f81..7d3a3b9 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -80,9 +80,9 @@ async def root(request: Request): return response -@app.route('/api') -@app.route('/api/') -async def api(request: Request): +@app.route('/openapi') +@app.route('/openapi/') +async def openapi(request: Request): """ OpenAPI access point @@ -91,7 +91,7 @@ async def api(request: Request): with open(os.environ.get('PYGEOAPI_OPENAPI')) as ff: openapi = yaml_load(ff) - headers, status_code, content = api_.api( + headers, status_code, content = api_.openapi( request.headers, request.query_params, openapi) response = Response(content=content, status_code=status_code) diff --git a/pygeoapi/templates/api.html b/pygeoapi/templates/openapi.html similarity index 100% rename from pygeoapi/templates/api.html rename to pygeoapi/templates/openapi.html diff --git a/pygeoapi/templates/root.html b/pygeoapi/templates/root.html index 6358ac2..365dd25 100644 --- a/pygeoapi/templates/root.html +++ b/pygeoapi/templates/root.html @@ -56,10 +56,10 @@ View the processes in this service

-
+

API Definition

- OpenAPI + OpenAPI

diff --git a/pygeoapi/util.py b/pygeoapi/util.py index fe37953..7d8b331 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2019 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -32,27 +32,34 @@ from datetime import date, datetime, time from decimal import Decimal import logging +import os +import re import yaml LOGGER = logging.getLogger(__name__) -def get_url(scheme, host, port, basepath): +def get_typed_value(value): """ - Provides URL of instance + Derive true type from data value - :returns: string of complete baseurl + :param value: value + + :returns: value as a native Python data type """ - url = '{}://{}'.format(scheme, host) + try: + if '.' in value: # float? + value2 = float(value) + elif len(value) > 1 and value.startswith('0'): + value2 = value + else: # int? + value2 = int(value) + except ValueError: # string (default)? + value2 = value - if port not in [80, 443]: - url = '{}:{}'.format(url, port) - - url = '{}{}'.format(url, basepath) - - return url + return value2 def yaml_load(fh): @@ -64,11 +71,23 @@ def yaml_load(fh): :returns: `dict` representation of YAML """ - try: - return yaml.load(fh, Loader=yaml.FullLoader) - except AttributeError as err: - LOGGER.warning('YAML loading error: {}'.format(err)) - return yaml.load(fh) + # support environment variables in config + # https://stackoverflow.com/a/55301129 + path_matcher = re.compile(r'.*\$\{([^}^{]+)\}.*') + + def path_constructor(loader, node): + env_var = path_matcher.match(node.value).group(1) + if env_var not in os.environ: + raise EnvironmentError('Undefined environment variable in config') + return get_typed_value(os.path.expandvars(node.value)) + + class EnvVarLoader(yaml.SafeLoader): + pass + + EnvVarLoader.add_implicit_resolver('!path', path_matcher, None) + EnvVarLoader.add_constructor('!path', path_constructor) + + return yaml.load(fh, Loader=EnvVarLoader) def str2bool(value): @@ -100,8 +119,7 @@ def json_serial(obj): """ if isinstance(obj, (datetime, date, time)): - serial = obj.isoformat() - return serial + return obj.isoformat() elif isinstance(obj, Decimal): return float(obj) diff --git a/tests/data/hotosm_bdi_waterways.sql.gz b/tests/data/hotosm_bdi_waterways.sql.gz index fbfe579..df0aeab 100644 Binary files a/tests/data/hotosm_bdi_waterways.sql.gz and b/tests/data/hotosm_bdi_waterways.sql.gz differ diff --git a/tests/pygeoapi-test-config-envvars.yml b/tests/pygeoapi-test-config-envvars.yml new file mode 100644 index 0000000..717f94f --- /dev/null +++ b/tests/pygeoapi-test-config-envvars.yml @@ -0,0 +1,120 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2019 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +server: + bind: + host: 0.0.0.0 + port: ${PYGEOAPI_PORT} + url: http://localhost:5000/ + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + language: en-US + cors: true + pretty_print: true + limit: 10 + # templates: /path/to/templates + map: + url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png + attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + +logging: + level: ERROR + #logfile: /tmp/pygeoapi.log + +metadata: + identification: + title: pygeoapi default instance ${PYGEOAPI_TITLE} + description: pygeoapi provides an API to geospatial data + keywords: + - geospatial + - data + - api + keywords_type: theme + terms_of_service: None + url: http://example.org + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: Organization Name + url: https://pygeoapi.io + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Hours of Service + instructions: During hours of service. Off on weekends. + role: pointOfContact + +datasets: + obs: + title: Observations + description: My cool observations + keywords: + - observations + - monitoring + links: + - type: text/csv + rel: canonical + title: data + href: https://github.com/mapserver/mapserver/blob/branch-7-0/msautotest/wxs/data/obs.csv + hreflang: en-US + - type: text/csv + rel: alternate + title: data + href: https://raw.githubusercontent.com/mapserver/mapserver/branch-7-0/msautotest/wxs/data/obs.csv + hreflang: en-US + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + temporal: + begin: 2000-10-30T18:24:39Z + end: 2007-10-30T08:57:29Z + trs: http://www.opengis.net/def/uom/ISO-8601/0/Gregorian + provider: + name: CSV + data: tests/data/obs.csv + id_field: id + geometry: + x_field: long + y_field: lat + +processes: + hello-world: + processor: + name: HelloWorld diff --git a/tests/pygeoapi-test-openapi.yml b/tests/pygeoapi-test-openapi.yml index 0a0a727..bb798bc 100644 --- a/tests/pygeoapi-test-openapi.yml +++ b/tests/pygeoapi-test-openapi.yml @@ -112,7 +112,7 @@ paths: summary: Landing page tags: - server - /api: + /openapi: get: description: This document parameters: diff --git a/tests/test_api.py b/tests/test_api.py index 8dd1d3f..fba43a4 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -77,7 +77,7 @@ def test_api(config, api_, openapi): assert isinstance(api_.config, dict) req_headers = make_req_headers(HTTP_CONTENT_TYPE='application/json') - rsp_headers, code, response = api_.api(req_headers, {}, openapi) + rsp_headers, code, response = api_.openapi(req_headers, {}, openapi) assert rsp_headers['Content-Type'] ==\ 'application/vnd.oai.openapi+json;version=3.0' root = json.loads(response) @@ -86,11 +86,12 @@ def test_api(config, api_, openapi): a = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' req_headers = make_req_headers(HTTP_ACCEPT=a) - rsp_headers, code, response = api_.api(req_headers, {}, openapi) + rsp_headers, code, response = api_.openapi(req_headers, {}, openapi) assert rsp_headers['Content-Type'] == 'text/html' req_headers = make_req_headers() - rsp_headers, code, response = api_.api(req_headers, {'f': 'foo'}, openapi) + rsp_headers, code, response = api_.openapi(req_headers, {'f': 'foo'}, + openapi) assert code == 400 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..7d2b455 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,62 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2019 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import os + +import pytest + +from pygeoapi.util import yaml_load + + +def get_test_file_path(filename): + """helper function to open test file safely""" + + if os.path.isfile(filename): + return filename + else: + return 'tests/{}'.format(filename) + + +def test_config_envvars(): + os.environ['PYGEOAPI_PORT'] = '5001' + os.environ['PYGEOAPI_TITLE'] = 'my title' + + with open(get_test_file_path('pygeoapi-test-config-envvars.yml')) as fh: + config = yaml_load(fh) + + assert isinstance(config, dict) + assert config['server']['bind']['port'] == 5001 + assert config['metadata']['identification']['title'] == \ + 'pygeoapi default instance my title' + + os.environ.pop('PYGEOAPI_PORT') + + with pytest.raises(EnvironmentError): + with open(get_test_file_path('pygeoapi-test-config-envvars.yml')) as fh: # noqa + config = yaml_load(fh) diff --git a/tests/test_postgresql_provider.py b/tests/test_postgresql_provider.py index 16f61a7..83d43c1 100644 --- a/tests/test_postgresql_provider.py +++ b/tests/test_postgresql_provider.py @@ -41,10 +41,12 @@ def config(): 'data': {'host': '127.0.0.1', 'dbname': 'test', 'user': 'postgres', - 'password': 'postgres' + 'password': 'postgres', + 'search_path': ['osm', 'public'] }, 'id_field': 'osm_id', - 'table': 'hotosm_bdi_waterways' + 'table': 'hotosm_bdi_waterways', + 'geom_field': 'foo_geom' } diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..f4b8aca --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,98 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2019 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +from datetime import datetime, date, time +from decimal import Decimal +import os + +import pytest + +from pygeoapi import util + + +def get_test_file_path(filename): + """helper function to open test file safely""" + + if os.path.isfile(filename): + return filename + else: + return 'tests/{}'.format(filename) + + +def test_get_typed_value(): + value = util.get_typed_value('2') + assert isinstance(value, int) + + value = util.get_typed_value('1.2') + assert isinstance(value, float) + + value = util.get_typed_value('1.c2') + assert isinstance(value, str) + + +def test_yaml_load(): + with open(get_test_file_path('pygeoapi-test-config.yml')) as fh: + d = util.yaml_load(fh) + assert isinstance(d, dict) + with pytest.raises(FileNotFoundError): + with open(get_test_file_path('404.yml')) as fh: + d = util.yaml_load(fh) + + +def test_str2bool(): + assert util.str2bool(False) is False + assert util.str2bool('0') is False + assert util.str2bool('no') is False + assert util.str2bool('yes') is True + assert util.str2bool('1') is True + assert util.str2bool(True) is True + assert util.str2bool('true') is True + assert util.str2bool('True') is True + assert util.str2bool('TRUE') is True + assert util.str2bool('tRuE') is True + assert util.str2bool('on') is True + assert util.str2bool('On') is True + assert util.str2bool('off') is False + + +def test_json_serial(): + d = datetime(1972, 10, 30) + assert util.json_serial(d) == '1972-10-30T00:00:00' + + d = date(2010, 7, 31) + assert util.json_serial(d) == '2010-07-31' + + d = time(11) + assert util.json_serial(d) == '11:00:00' + + d = Decimal(1.0) + assert util.json_serial(d) == 1.0 + + with pytest.raises(TypeError): + util.json_serial('foo')