From 01bcf8084b57dc2f91b4349f1315b7a0bb0cba53 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 19 Feb 2019 18:51:49 -0500 Subject: [PATCH] plugin refactor (#83) --- pygeoapi/api.py | 15 +++--- pygeoapi/formatter/__init__.py | 47 +---------------- pygeoapi/formatter/base.py | 13 ++--- pygeoapi/formatter/csv_.py | 10 ++-- pygeoapi/openapi.py | 6 +-- pygeoapi/plugin.py | 92 +++++++++++++++++++++++++++++++++ pygeoapi/provider/__init__.py | 53 +------------------ pygeoapi/provider/geopackage.py | 12 ++--- pygeoapi/provider/sqlite.py | 8 +-- tests/test_csv_formatter.py | 4 +- 10 files changed, 131 insertions(+), 129 deletions(-) create mode 100644 pygeoapi/plugin.py diff --git a/pygeoapi/api.py b/pygeoapi/api.py index 89a759a..8fac672 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2019 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -36,8 +36,7 @@ from jinja2 import Environment, FileSystemLoader from pygeoapi import __version__ from pygeoapi.log import setup_logger -from pygeoapi.provider import load_provider -from pygeoapi.formatter import FORMATTERS, load_formatter +from pygeoapi.plugin import load_plugin, PLUGINS from pygeoapi.provider.base import ProviderConnectionError, ProviderQueryError LOGGER = logging.getLogger(__name__) @@ -283,7 +282,7 @@ class API(object): reserved_fieldnames = ['bbox', 'f', 'limit', 'startindex', 'resulttype', 'time'] formats = ['json', 'html'] - formats.extend(f.lower() for f in FORMATTERS.keys()) + formats.extend(f.lower() for f in PLUGINS['formatter'].keys()) if dataset not in self.config['datasets'].keys(): exception = { @@ -330,7 +329,8 @@ class API(object): LOGGER.debug('Loading provider') try: - p = load_provider(self.config['datasets'][dataset]['provider']) + p = load_plugin('provider', + self.config['datasets'][dataset]['provider']) except ProviderConnectionError: exception = { 'code': 'NoApplicableCode', @@ -447,7 +447,7 @@ class API(object): content) return headers_, 200, content elif format_ == 'csv': # render - formatter = load_formatter('CSV', geom=True) + formatter = load_plugin('formatter', {'name': 'CSV', 'geom': True}) content = formatter.write( data=content, @@ -503,7 +503,8 @@ class API(object): return headers_, 400, json.dumps(exception) LOGGER.debug('Loading provider') - p = load_provider(self.config['datasets'][dataset]['provider']) + p = load_plugin('provider', + self.config['datasets'][dataset]['provider']) LOGGER.debug('Fetching id {}'.format(identifier)) content = p.get(identifier) diff --git a/pygeoapi/formatter/__init__.py b/pygeoapi/formatter/__init__.py index dc192ae..71c632c 100644 --- a/pygeoapi/formatter/__init__.py +++ b/pygeoapi/formatter/__init__.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2019 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -26,48 +26,3 @@ # OTHER DEALINGS IN THE SOFTWARE. # # ================================================================= - -import importlib -import logging - -LOGGER = logging.getLogger(__name__) - -FORMATTERS = { - 'CSV': 'pygeoapi.formatter.csv_.CSVFormatter', -} - - -def load_formatter(name, geom=False): - """ - loads formatter by name - - :param name: formatter name - :param geom: whether to emit geometry (default False) - - :returns: formatter object - """ - - LOGGER.debug('Formatters: {}'.format(FORMATTERS)) - - if '.' not in name and name not in FORMATTERS.keys(): - msg = 'Formatter {} not found'.format(name) - LOGGER.exception(msg) - raise InvalidFormatterError(msg) - - if '.' in name: # dotted path - packagename, classname = name.rsplit('.', 1) - else: # core formatter - packagename, classname = FORMATTERS[name].rsplit('.', 1) - - LOGGER.debug('package name: {}'.format(packagename)) - LOGGER.debug('class name: {}'.format(classname)) - - module = importlib.import_module(packagename) - class_ = getattr(module, classname) - formatter = class_(geom) - return formatter - - -class InvalidFormatterError(Exception): - """Invalid formatter""" - pass diff --git a/pygeoapi/formatter/base.py b/pygeoapi/formatter/base.py index b7be5b8..a0c761e 100644 --- a/pygeoapi/formatter/base.py +++ b/pygeoapi/formatter/base.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2019 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -35,20 +35,21 @@ LOGGER = logging.getLogger(__name__) class BaseFormatter(object): """generic Formatter ABC""" - def __init__(self, name, geom=False): + def __init__(self, formatter_def): """ Initialize object - :param name: formatter name - :param geom: whether to emit geometry (default False) + :param formatter_def: formatter definition :returns: pygeoapi.providers.base.BaseFormatter """ self.mimetype = None + self.geom = False - self.name = name - self.geom = geom + self.name = formatter_def['name'] + if 'geom' in formatter_def: + self.geom = formatter_def['geom'] def write(self, options={}, data=None): """ diff --git a/pygeoapi/formatter/csv_.py b/pygeoapi/formatter/csv_.py index d035e1b..6eaf253 100644 --- a/pygeoapi/formatter/csv_.py +++ b/pygeoapi/formatter/csv_.py @@ -40,16 +40,20 @@ LOGGER = logging.getLogger(__name__) class CSVFormatter(BaseFormatter): """CSV formatter""" - def __init__(self, geom=False): + def __init__(self, formatter_def): """ Initialize object - :param geom: whether to emit geometry (default False) + :param formatter_def: formatter definition :returns: pygeoapi.formatter.csv_.CSVFormatter """ - BaseFormatter.__init__(self, 'csv', geom) + geom = False + if 'geom' in formatter_def: + geom = formatter_def['geom'] + + BaseFormatter.__init__(self, {'name': 'csv', 'geom': geom}) self.mimetype = 'text/csv' def write(self, options={}, data=None): diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index 134ffe6..5013f8a 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2019 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -32,7 +32,7 @@ import logging import click import yaml -from pygeoapi.provider import load_provider +from pygeoapi.plugin import load_plugin LOGGER = logging.getLogger(__name__) @@ -195,7 +195,7 @@ def get_oas_30(cfg): } } - p = load_provider(cfg['datasets'][k]['provider']) + p = load_plugin('provider', cfg['datasets'][k]['provider']) for k2, v2 in p.fields.items(): path_ = '{}/items'.format(collection_name_path) diff --git a/pygeoapi/plugin.py b/pygeoapi/plugin.py new file mode 100644 index 0000000..ae572a8 --- /dev/null +++ b/pygeoapi/plugin.py @@ -0,0 +1,92 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2019 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import importlib +import logging + +LOGGER = logging.getLogger(__name__) + +PLUGINS = { + 'provider': { + 'CSV': 'pygeoapi.provider.csv_.CSVProvider', + 'Elasticsearch': 'pygeoapi.provider.elasticsearch_.ElasticsearchProvider', # noqa + 'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider', + 'GeoPackage': 'pygeoapi.provider.geopackage.GeoPackageProvider', + 'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider', + 'SQLite': 'pygeoapi.provider.sqlite.SQLiteProvider' + }, + 'formatter': { + 'CSV': 'pygeoapi.formatter.csv_.CSVFormatter' + } +} + + +def load_plugin(plugin_type, plugin_def): + """ + loads plugin by name + + :param plugin_type: type of plugin (provider, formatter) + :param plugin_def: plugin definition + + :returns: plugin object + """ + + name = plugin_def['name'] + + if plugin_type not in PLUGINS.keys(): + msg = 'Plugin type {} not found'.format(plugin_type) + LOGGER.exception(msg) + raise InvalidPluginError(msg) + + plugin_list = PLUGINS[plugin_type] + + LOGGER.debug('Plugins: {}'.format(plugin_list)) + + if '.' not in name and name not in plugin_list.keys(): + msg = 'Plugin {} not found'.format(name) + LOGGER.exception(msg) + raise InvalidPluginError(msg) + + if '.' in name: # dotted path + packagename, classname = name.rsplit('.', 1) + else: # core formatter + packagename, classname = plugin_list[name].rsplit('.', 1) + + LOGGER.debug('package name: {}'.format(packagename)) + LOGGER.debug('class name: {}'.format(classname)) + + module = importlib.import_module(packagename) + class_ = getattr(module, classname) + plugin = class_(plugin_def) + return plugin + + +class InvalidPluginError(Exception): + """Invalid plugin""" + pass diff --git a/pygeoapi/provider/__init__.py b/pygeoapi/provider/__init__.py index a487a21..71c632c 100644 --- a/pygeoapi/provider/__init__.py +++ b/pygeoapi/provider/__init__.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2019 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -26,54 +26,3 @@ # OTHER DEALINGS IN THE SOFTWARE. # # ================================================================= - -import importlib -import logging - -LOGGER = logging.getLogger(__name__) - -PROVIDERS = { - 'CSV': 'pygeoapi.provider.csv_.CSVProvider', - 'Elasticsearch': 'pygeoapi.provider.elasticsearch_.ElasticsearchProvider', - 'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider', - 'GeoPackage': 'pygeoapi.provider.geopackage.GeoPackageProvider', - 'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider', - 'SQLite': 'pygeoapi.provider.sqlite.SQLiteProvider' -} - - -def load_provider(provider_def): - """ - loads provider by name - - :param provider_def: provider definition - - :returns: provider object - """ - - LOGGER.debug('Providers: {}'.format(PROVIDERS)) - - pname = provider_def['name'] - - if '.' not in pname and pname not in PROVIDERS.keys(): - msg = 'Provider {} not found'.format(pname) - LOGGER.exception(msg) - raise InvalidProviderError(msg) - - if '.' in pname: # dotted path - packagename, classname = pname.rsplit('.', 1) - else: # core provider - packagename, classname = PROVIDERS[pname].rsplit('.', 1) - - LOGGER.debug('package name: {}'.format(packagename)) - LOGGER.debug('class name: {}'.format(classname)) - - module = importlib.import_module(packagename) - class_ = getattr(module, classname) - provider = class_(provider_def) - return provider - - -class InvalidProviderError(Exception): - """invalid provider""" - pass diff --git a/pygeoapi/provider/geopackage.py b/pygeoapi/provider/geopackage.py index f314d12..dac3445 100644 --- a/pygeoapi/provider/geopackage.py +++ b/pygeoapi/provider/geopackage.py @@ -31,8 +31,8 @@ import sqlite3 import logging import os import json +from pygeoapi.plugin import InvalidPluginError from pygeoapi.provider.base import BaseProvider, ProviderConnectionError -from pygeoapi.provider import InvalidProviderError LOGGER = logging.getLogger(__name__) @@ -118,7 +118,7 @@ class GeoPackageProvider(BaseProvider): if (os.path.exists(self.data)): conn = sqlite3.connect(self.data) else: - raise InvalidProviderError + raise InvalidPluginError try: conn.enable_load_extension(True) @@ -145,15 +145,15 @@ class GeoPackageProvider(BaseProvider): else: LOGGER.info("SELECT AutoGPKGStart() returned 0." + "Likely that this is not a GeoPackage") - raise InvalidProviderError - except InvalidProviderError: + raise InvalidPluginError + except InvalidPluginError: raise cursor.execute("PRAGMA table_info({})".format(self.view)) result = cursor.fetchall() try: # TODO: Better exceptions declaring - # InvalidProviderError as Parent class + # InvalidPluginError as Parent class assert len(result), "Table not found" assert len([item for item in result if self.id_field in item]), "id_field not present" @@ -162,7 +162,7 @@ class GeoPackageProvider(BaseProvider): assert len([item for item in result if 'geom' in item]), "geom column not found" - except InvalidProviderError: + except InvalidPluginError: raise self.columns = [item[1] for item in result if item[1] != 'geom'] diff --git a/pygeoapi/provider/sqlite.py b/pygeoapi/provider/sqlite.py index 4d6ad03..63e0921 100644 --- a/pygeoapi/provider/sqlite.py +++ b/pygeoapi/provider/sqlite.py @@ -31,8 +31,8 @@ import sqlite3 import logging import os import json +from pygeoapi.plugin import InvalidPluginError from pygeoapi.provider.base import BaseProvider, ProviderConnectionError -from pygeoapi.provider import InvalidProviderError LOGGER = logging.getLogger(__name__) @@ -113,7 +113,7 @@ class SQLiteProvider(BaseProvider): if (os.path.exists(self.data)): conn = sqlite3.connect(self.data) else: - raise InvalidProviderError + raise InvalidPluginError try: conn.enable_load_extension(True) @@ -133,7 +133,7 @@ class SQLiteProvider(BaseProvider): result = cursor.fetchall() try: # TODO: Better exceptions declaring - # InvalidProviderError as Parent class + # InvalidPluginError as Parent class assert len(result), "Table not found" assert len([item for item in result if item['pk'] == 1]), "Primary key not found" @@ -142,7 +142,7 @@ class SQLiteProvider(BaseProvider): assert len([item for item in result if 'GEOMETRY' in item]), "GEOMETRY column not found" - except InvalidProviderError: + except InvalidPluginError: raise self.columns = [item[1] for item in result if item[1] != 'GEOMETRY'] diff --git a/tests/test_csv_formatter.py b/tests/test_csv_formatter.py index e776114..bc71931 100644 --- a/tests/test_csv_formatter.py +++ b/tests/test_csv_formatter.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2019 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -59,7 +59,7 @@ def fixture(): def test_csv_formatter(fixture): - f = CSVFormatter(geom=True) + f = CSVFormatter({'geom': True}) f_csv = f.write(data=fixture) buffer = io.StringIO(f_csv.decode('utf-8'))