From 081d7ed1c8450adf62825ed3ac355f0da0f02e8a Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Wed, 4 Jul 2018 06:29:05 -0400 Subject: [PATCH] add support for CSV output (#52) --- debian/control | 2 +- pygeoapi/api.py | 17 ++++++ pygeoapi/formatters/__init__.py | 73 ++++++++++++++++++++++++ pygeoapi/formatters/base.py | 66 ++++++++++++++++++++++ pygeoapi/formatters/csv_.py | 98 +++++++++++++++++++++++++++++++++ requirements.txt | 1 + tests/test_csv_formatter.py | 80 +++++++++++++++++++++++++++ 7 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 pygeoapi/formatters/__init__.py create mode 100644 pygeoapi/formatters/base.py create mode 100644 pygeoapi/formatters/csv_.py create mode 100644 tests/test_csv_formatter.py diff --git a/debian/control b/debian/control index be0ad48..f4576aa 100644 --- a/debian/control +++ b/debian/control @@ -9,7 +9,7 @@ Vcs-Git: https://github.com/geopython/pygeoapi.git Package: python-pygeoapi Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, python-pkg-resources, python-click, python-flask, python-yaml +Depends: ${misc:Depends}, ${python:Depends}, python-pkg-resources, python-click, python-flask, python-unicodecsv, python-yaml Suggests: python-elasticsearch, python-geojson Homepage: https://github.com/geopython/pygeoapi Description: pygeoapi provides an API to geospatial diff --git a/pygeoapi/api.py b/pygeoapi/api.py index f57163c..2c63588 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -37,6 +37,7 @@ from jinja2 import Environment, FileSystemLoader from pygeoapi import __version__ from pygeoapi.log import setup_logger from pygeoapi.provider import load_provider +from pygeoapi.formatters import FORMATTERS, load_formatter from pygeoapi.provider.base import ProviderConnectionError, ProviderQueryError LOGGER = logging.getLogger(__name__) @@ -286,6 +287,7 @@ class API(object): reserved_fieldnames = ['bbox', 'f', 'limit', 'startindex', 'resulttype', 'time'] formats = ['json', 'html'] + formats.extend(f.lower() for f in FORMATTERS.keys()) if dataset not in self.config['datasets'].keys(): exception = { @@ -417,6 +419,21 @@ class API(object): content = _render_j2_template(self.config, 'items.html', content) return headers_, 200, content + elif format_ == 'csv': # render + formatter = load_formatter('CSV', geom=True) + + content = formatter.write( + data=content, + options={ + 'provider_def': + self.config['datasets'][dataset]['provider'] + } + ) + + headers_['Content-type'] = '{}; charset={}'.format( + formatter.mimetype, self.config['server']['encoding']) + + return headers_, 200, content return headers_, 200, json.dumps(content) diff --git a/pygeoapi/formatters/__init__.py b/pygeoapi/formatters/__init__.py new file mode 100644 index 0000000..1686c56 --- /dev/null +++ b/pygeoapi/formatters/__init__.py @@ -0,0 +1,73 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2018 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import importlib +import logging + +LOGGER = logging.getLogger(__name__) + +FORMATTERS = { + 'CSV': 'pygeoapi.formatters.csv_.CSVFormatter', +} + + +def load_formatter(name, geom=False): + """ + loads formatter by name + + :param name: formatter name + :param geom: whether to emit geometry (default False) + + :returns: formatter object + """ + + LOGGER.debug('Formatters: {}'.format(FORMATTERS)) + + if '.' not in name and name not in FORMATTERS.keys(): + msg = 'Formatter {} not found'.format(name) + LOGGER.exception(msg) + raise InvalidFormatterError(msg) + + if '.' in name: # dotted path + packagename, classname = name.rsplit('.', 1) + else: # core formatter + packagename, classname = FORMATTERS[name].rsplit('.', 1) + + LOGGER.debug('package name: {}'.format(packagename)) + LOGGER.debug('class name: {}'.format(classname)) + + module = importlib.import_module(packagename) + class_ = getattr(module, classname) + formatter = class_(geom) + return formatter + + +class InvalidFormatterError(Exception): + """Invalid formatter""" + pass diff --git a/pygeoapi/formatters/base.py b/pygeoapi/formatters/base.py new file mode 100644 index 0000000..b7be5b8 --- /dev/null +++ b/pygeoapi/formatters/base.py @@ -0,0 +1,66 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2018 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging + +LOGGER = logging.getLogger(__name__) + + +class BaseFormatter(object): + """generic Formatter ABC""" + + def __init__(self, name, geom=False): + """ + Initialize object + + :param name: formatter name + :param geom: whether to emit geometry (default False) + + :returns: pygeoapi.providers.base.BaseFormatter + """ + + self.mimetype = None + + self.name = name + self.geom = geom + + def write(self, options={}, data=None): + """ + Generate data in specified format + + :param options: CSV formatting options + :param data: dict representation of GeoJSON object + + :returns: string representation of format + """ + + raise NotImplementedError() + + def __repr__(self): + return ' {}'.format(self.name) diff --git a/pygeoapi/formatters/csv_.py b/pygeoapi/formatters/csv_.py new file mode 100644 index 0000000..e912972 --- /dev/null +++ b/pygeoapi/formatters/csv_.py @@ -0,0 +1,98 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2018 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import io +import logging + +import unicodecsv as csv + +from pygeoapi.formatters.base import BaseFormatter + +LOGGER = logging.getLogger(__name__) + + +class CSVFormatter(BaseFormatter): + """CSV formatter""" + + def __init__(self, geom=False): + """ + Initialize object + + :param geom: whether to emit geometry (default False) + + :returns: pygeoapi.formatters.csv_.CSVFormatter + """ + + BaseFormatter.__init__(self, 'csv', geom) + self.mimetype = 'text/plain' + + def write(self, options={}, data=None): + """ + Generate data in CSV format + + :param options: CSV formatting options + :param data: dict of GeoJSON data + + :returns: string representation of format + """ + + is_point = False + try: + fields = list(data['features'][0]['properties'].keys()) + except IndexError: + LOGGER.error('no features') + return str() + + if self.geom: + LOGGER.debug('Including point geometry') + if data['features'][0]['geometry']['type'] == 'Point': + fields.insert(0, 'x') + fields.insert(1, 'y') + is_point = True + else: + # TODO: implement wkt geometry serialization + LOGGER.debug('not a point geometry, skipping') + + LOGGER.debug('CSV fields: {}'.format(fields)) + + output = io.BytesIO() + writer = csv.DictWriter(output, fields) + writer.writeheader() + + for feature in data['features']: + fp = feature['properties'] + if is_point: + fp['x'] = feature['geometry']['coordinates'][0] + fp['y'] = feature['geometry']['coordinates'][1] + LOGGER.debug(fp) + writer.writerow(fp) + return output.getvalue() + + def __repr__(self): + return ' {}'.format(self.mimetype) diff --git a/requirements.txt b/requirements.txt index 0fa0a4b..b49283d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ click Flask PyYAML +unicodecsv diff --git a/tests/test_csv_formatter.py b/tests/test_csv_formatter.py new file mode 100644 index 0000000..cbdc278 --- /dev/null +++ b/tests/test_csv_formatter.py @@ -0,0 +1,80 @@ +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2018 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import csv +import io +import pytest + +from pygeoapi.formatters.csv_ import CSVFormatter + + +@pytest.fixture() +def fixture(): + data = { + 'features': [{ + 'geometry': { + 'type': 'Point', + 'coordinates': [ + -130.44472222222223, + 54.28611111111111 + ] + }, + 'type': 'Feature', + 'properties': { + 'id': 1972, + 'foo': 'bar', + 'title': None, + }, + 'ID': 48693 + }] + } + + return data + + +def test_csv_formatter(fixture): + f = CSVFormatter(geom=True) + f_csv = f.write(data=fixture) + + buffer = io.StringIO(f_csv.decode('utf-8')) + reader = csv.DictReader(buffer) + + header = list(reader.fieldnames) + + assert len(header) == 5 + + assert 'x' in header + assert 'y' in header + + data = next(reader) + assert data['x'] == '-130.44472222222223' + assert data['y'] == '54.28611111111111' + assert data['id'] == '1972' + assert data['foo'] == 'bar' + assert data['title'] == ''