huanld/pygeoapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

plugin refactor (#83)

Browse Source
This commit is contained in:
Tom Kralidis
2019-02-19 18:51:49 -05:00
committed by GitHub
parent fd761a967d
commit 01bcf8084b
10 changed files with 131 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pygeoapi/api.py
+8 -7
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
# Copyright (c) 2019 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -36,8 +36,7 @@ from jinja2 import Environment, FileSystemLoader
from pygeoapi import __version__
from pygeoapi.log import setup_logger
from pygeoapi.provider import load_provider
from pygeoapi.formatter import FORMATTERS, load_formatter
from pygeoapi.plugin import load_plugin, PLUGINS
from pygeoapi.provider.base import ProviderConnectionError, ProviderQueryError
LOGGER = logging.getLogger(__name__)
@@ -283,7 +282,7 @@ class API(object):
reserved_fieldnames = ['bbox', 'f', 'limit', 'startindex',
'resulttype', 'time']
formats = ['json', 'html']
formats.extend(f.lower() for f in FORMATTERS.keys())
formats.extend(f.lower() for f in PLUGINS['formatter'].keys())
if dataset not in self.config['datasets'].keys():
exception = {
@@ -330,7 +329,8 @@ class API(object):
LOGGER.debug('Loading provider')
try:
p = load_provider(self.config['datasets'][dataset]['provider'])
p = load_plugin('provider',
self.config['datasets'][dataset]['provider'])
except ProviderConnectionError:
exception = {
'code': 'NoApplicableCode',
@@ -447,7 +447,7 @@ class API(object):
content)
return headers_, 200, content
elif format_ == 'csv': # render
formatter = load_formatter('CSV', geom=True)
formatter = load_plugin('formatter', {'name': 'CSV', 'geom': True})
content = formatter.write(
data=content,
@@ -503,7 +503,8 @@ class API(object):
return headers_, 400, json.dumps(exception)
LOGGER.debug('Loading provider')
p = load_provider(self.config['datasets'][dataset]['provider'])
p = load_plugin('provider',
self.config['datasets'][dataset]['provider'])
LOGGER.debug('Fetching id {}'.format(identifier))
content = p.get(identifier)
pygeoapi/formatter/__init__.py
+1 -46
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
# Copyright (c) 2019 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -26,48 +26,3 @@
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
import importlib
import logging
LOGGER = logging.getLogger(__name__)
FORMATTERS = {
'CSV': 'pygeoapi.formatter.csv_.CSVFormatter',
}
def load_formatter(name, geom=False):
"""
loads formatter by name
:param name: formatter name
:param geom: whether to emit geometry (default False)
:returns: formatter object
"""
LOGGER.debug('Formatters: {}'.format(FORMATTERS))
if '.' not in name and name not in FORMATTERS.keys():
msg = 'Formatter {} not found'.format(name)
LOGGER.exception(msg)
raise InvalidFormatterError(msg)
if '.' in name: # dotted path
packagename, classname = name.rsplit('.', 1)
else: # core formatter
packagename, classname = FORMATTERS[name].rsplit('.', 1)
LOGGER.debug('package name: {}'.format(packagename))
LOGGER.debug('class name: {}'.format(classname))
module = importlib.import_module(packagename)
class_ = getattr(module, classname)
formatter = class_(geom)
return formatter
class InvalidFormatterError(Exception):
"""Invalid formatter"""
pass
pygeoapi/formatter/base.py
+7 -6
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
# Copyright (c) 2019 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -35,20 +35,21 @@ LOGGER = logging.getLogger(__name__)
class BaseFormatter(object):
"""generic Formatter ABC"""
def __init__(self, name, geom=False):
def __init__(self, formatter_def):
"""
Initialize object
:param name: formatter name
:param geom: whether to emit geometry (default False)
:param formatter_def: formatter definition
:returns: pygeoapi.providers.base.BaseFormatter
"""
self.mimetype = None
self.geom = False
self.name = name
self.geom = geom
self.name = formatter_def['name']
if 'geom' in formatter_def:
self.geom = formatter_def['geom']
def write(self, options={}, data=None):
"""
pygeoapi/formatter/csv_.py
+7 -3
View File
@@ -40,16 +40,20 @@ LOGGER = logging.getLogger(__name__)
class CSVFormatter(BaseFormatter):
"""CSV formatter"""
def __init__(self, geom=False):
def __init__(self, formatter_def):
"""
Initialize object
:param geom: whether to emit geometry (default False)
:param formatter_def: formatter definition
:returns: pygeoapi.formatter.csv_.CSVFormatter
"""
BaseFormatter.__init__(self, 'csv', geom)
geom = False
if 'geom' in formatter_def:
geom = formatter_def['geom']
BaseFormatter.__init__(self, {'name': 'csv', 'geom': geom})
self.mimetype = 'text/csv'
def write(self, options={}, data=None):
pygeoapi/openapi.py
+3 -3
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
# Copyright (c) 2019 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -32,7 +32,7 @@ import logging
import click
import yaml
from pygeoapi.provider import load_provider
from pygeoapi.plugin import load_plugin
LOGGER = logging.getLogger(__name__)
@@ -195,7 +195,7 @@ def get_oas_30(cfg):
}
}
p = load_provider(cfg['datasets'][k]['provider'])
p = load_plugin('provider', cfg['datasets'][k]['provider'])
for k2, v2 in p.fields.items():
path_ = '{}/items'.format(collection_name_path)
pygeoapi/plugin.py
+92
View File
@@ -0,0 +1,92 @@
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2019 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
import importlib
import logging
LOGGER = logging.getLogger(__name__)
PLUGINS = {
'provider': {
'CSV': 'pygeoapi.provider.csv_.CSVProvider',
'Elasticsearch': 'pygeoapi.provider.elasticsearch_.ElasticsearchProvider', # noqa
'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider',
'GeoPackage': 'pygeoapi.provider.geopackage.GeoPackageProvider',
'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider',
'SQLite': 'pygeoapi.provider.sqlite.SQLiteProvider'
},
'formatter': {
'CSV': 'pygeoapi.formatter.csv_.CSVFormatter'
}
}
def load_plugin(plugin_type, plugin_def):
"""
loads plugin by name
:param plugin_type: type of plugin (provider, formatter)
:param plugin_def: plugin definition
:returns: plugin object
"""
name = plugin_def['name']
if plugin_type not in PLUGINS.keys():
msg = 'Plugin type {} not found'.format(plugin_type)
LOGGER.exception(msg)
raise InvalidPluginError(msg)
plugin_list = PLUGINS[plugin_type]
LOGGER.debug('Plugins: {}'.format(plugin_list))
if '.' not in name and name not in plugin_list.keys():
msg = 'Plugin {} not found'.format(name)
LOGGER.exception(msg)
raise InvalidPluginError(msg)
if '.' in name: # dotted path
packagename, classname = name.rsplit('.', 1)
else: # core formatter
packagename, classname = plugin_list[name].rsplit('.', 1)
LOGGER.debug('package name: {}'.format(packagename))
LOGGER.debug('class name: {}'.format(classname))
module = importlib.import_module(packagename)
class_ = getattr(module, classname)
plugin = class_(plugin_def)
return plugin
class InvalidPluginError(Exception):
"""Invalid plugin"""
pass
pygeoapi/provider/__init__.py
+1 -52
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
# Copyright (c) 2019 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -26,54 +26,3 @@
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
import importlib
import logging
LOGGER = logging.getLogger(__name__)
PROVIDERS = {
'CSV': 'pygeoapi.provider.csv_.CSVProvider',
'Elasticsearch': 'pygeoapi.provider.elasticsearch_.ElasticsearchProvider',
'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider',
'GeoPackage': 'pygeoapi.provider.geopackage.GeoPackageProvider',
'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider',
'SQLite': 'pygeoapi.provider.sqlite.SQLiteProvider'
}
def load_provider(provider_def):
"""
loads provider by name
:param provider_def: provider definition
:returns: provider object
"""
LOGGER.debug('Providers: {}'.format(PROVIDERS))
pname = provider_def['name']
if '.' not in pname and pname not in PROVIDERS.keys():
msg = 'Provider {} not found'.format(pname)
LOGGER.exception(msg)
raise InvalidProviderError(msg)
if '.' in pname: # dotted path
packagename, classname = pname.rsplit('.', 1)
else: # core provider
packagename, classname = PROVIDERS[pname].rsplit('.', 1)
LOGGER.debug('package name: {}'.format(packagename))
LOGGER.debug('class name: {}'.format(classname))
module = importlib.import_module(packagename)
class_ = getattr(module, classname)
provider = class_(provider_def)
return provider
class InvalidProviderError(Exception):
"""invalid provider"""
pass
pygeoapi/provider/geopackage.py
+6 -6
View File
@@ -31,8 +31,8 @@ import sqlite3
import logging
import os
import json
from pygeoapi.plugin import InvalidPluginError
from pygeoapi.provider.base import BaseProvider, ProviderConnectionError
from pygeoapi.provider import InvalidProviderError
LOGGER = logging.getLogger(__name__)
@@ -118,7 +118,7 @@ class GeoPackageProvider(BaseProvider):
if (os.path.exists(self.data)):
conn = sqlite3.connect(self.data)
else:
raise InvalidProviderError
raise InvalidPluginError
try:
conn.enable_load_extension(True)
@@ -145,15 +145,15 @@ class GeoPackageProvider(BaseProvider):
else:
LOGGER.info("SELECT AutoGPKGStart() returned 0." +
"Likely that this is not a GeoPackage")
raise InvalidProviderError
except InvalidProviderError:
raise InvalidPluginError
except InvalidPluginError:
raise
cursor.execute("PRAGMA table_info({})".format(self.view))
result = cursor.fetchall()
try:
# TODO: Better exceptions declaring
# InvalidProviderError as Parent class
# InvalidPluginError as Parent class
assert len(result), "Table not found"
assert len([item for item in result
if self.id_field in item]), "id_field not present"
@@ -162,7 +162,7 @@ class GeoPackageProvider(BaseProvider):
assert len([item for item in result
if 'geom' in item]), "geom column not found"
except InvalidProviderError:
except InvalidPluginError:
raise
self.columns = [item[1] for item in result if item[1] != 'geom']
pygeoapi/provider/sqlite.py
+4 -4
View File
@@ -31,8 +31,8 @@ import sqlite3
import logging
import os
import json
from pygeoapi.plugin import InvalidPluginError
from pygeoapi.provider.base import BaseProvider, ProviderConnectionError
from pygeoapi.provider import InvalidProviderError
LOGGER = logging.getLogger(__name__)
@@ -113,7 +113,7 @@ class SQLiteProvider(BaseProvider):
if (os.path.exists(self.data)):
conn = sqlite3.connect(self.data)
else:
raise InvalidProviderError
raise InvalidPluginError
try:
conn.enable_load_extension(True)
@@ -133,7 +133,7 @@ class SQLiteProvider(BaseProvider):
result = cursor.fetchall()
try:
# TODO: Better exceptions declaring
# InvalidProviderError as Parent class
# InvalidPluginError as Parent class
assert len(result), "Table not found"
assert len([item for item in result
if item['pk'] == 1]), "Primary key not found"
@@ -142,7 +142,7 @@ class SQLiteProvider(BaseProvider):
assert len([item for item in result
if 'GEOMETRY' in item]), "GEOMETRY column not found"
except InvalidProviderError:
except InvalidPluginError:
raise
self.columns = [item[1] for item in result if item[1] != 'GEOMETRY']
tests/test_csv_formatter.py
+2 -2
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
# Copyright (c) 2019 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -59,7 +59,7 @@ def fixture():
def test_csv_formatter(fixture):
f = CSVFormatter(geom=True)
f = CSVFormatter({'geom': True})
f_csv = f.write(data=fixture)
buffer = io.StringIO(f_csv.decode('utf-8'))