huanld/pygeoapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add support for CSV output (#52)

Browse Source
This commit is contained in:
Tom Kralidis
2018-07-04 06:29:05 -04:00
committed by GitHub
parent c1e9943c48
commit 081d7ed1c8
7 changed files with 336 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
debian/control
Vendored
+1 -1
View File
@@ -9,7 +9,7 @@ Vcs-Git: https://github.com/geopython/pygeoapi.git
Package: python-pygeoapi
Architecture: all
Depends: ${misc:Depends}, ${python:Depends}, python-pkg-resources, python-click, python-flask, python-yaml
Depends: ${misc:Depends}, ${python:Depends}, python-pkg-resources, python-click, python-flask, python-unicodecsv, python-yaml
Suggests: python-elasticsearch, python-geojson
Homepage: https://github.com/geopython/pygeoapi
Description: pygeoapi provides an API to geospatial
pygeoapi/api.py
+17
View File
@@ -37,6 +37,7 @@ from jinja2 import Environment, FileSystemLoader
from pygeoapi import __version__
from pygeoapi.log import setup_logger
from pygeoapi.provider import load_provider
from pygeoapi.formatters import FORMATTERS, load_formatter
from pygeoapi.provider.base import ProviderConnectionError, ProviderQueryError
LOGGER = logging.getLogger(__name__)
@@ -286,6 +287,7 @@ class API(object):
reserved_fieldnames = ['bbox', 'f', 'limit', 'startindex',
'resulttype', 'time']
formats = ['json', 'html']
formats.extend(f.lower() for f in FORMATTERS.keys())
if dataset not in self.config['datasets'].keys():
exception = {
@@ -417,6 +419,21 @@ class API(object):
content = _render_j2_template(self.config, 'items.html',
content)
return headers_, 200, content
elif format_ == 'csv': # render
formatter = load_formatter('CSV', geom=True)
content = formatter.write(
data=content,
options={
'provider_def':
self.config['datasets'][dataset]['provider']
}
)
headers_['Content-type'] = '{}; charset={}'.format(
formatter.mimetype, self.config['server']['encoding'])
return headers_, 200, content
return headers_, 200, json.dumps(content)
pygeoapi/formatters/__init__.py
+73
View File
@@ -0,0 +1,73 @@
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
import importlib
import logging
LOGGER = logging.getLogger(__name__)
FORMATTERS = {
'CSV': 'pygeoapi.formatters.csv_.CSVFormatter',
}
def load_formatter(name, geom=False):
"""
loads formatter by name
:param name: formatter name
:param geom: whether to emit geometry (default False)
:returns: formatter object
"""
LOGGER.debug('Formatters: {}'.format(FORMATTERS))
if '.' not in name and name not in FORMATTERS.keys():
msg = 'Formatter {} not found'.format(name)
LOGGER.exception(msg)
raise InvalidFormatterError(msg)
if '.' in name: # dotted path
packagename, classname = name.rsplit('.', 1)
else: # core formatter
packagename, classname = FORMATTERS[name].rsplit('.', 1)
LOGGER.debug('package name: {}'.format(packagename))
LOGGER.debug('class name: {}'.format(classname))
module = importlib.import_module(packagename)
class_ = getattr(module, classname)
formatter = class_(geom)
return formatter
class InvalidFormatterError(Exception):
"""Invalid formatter"""
pass
pygeoapi/formatters/base.py
+66
View File
@@ -0,0 +1,66 @@
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
import logging
LOGGER = logging.getLogger(__name__)
class BaseFormatter(object):
"""generic Formatter ABC"""
def __init__(self, name, geom=False):
"""
Initialize object
:param name: formatter name
:param geom: whether to emit geometry (default False)
:returns: pygeoapi.providers.base.BaseFormatter
"""
self.mimetype = None
self.name = name
self.geom = geom
def write(self, options={}, data=None):
"""
Generate data in specified format
:param options: CSV formatting options
:param data: dict representation of GeoJSON object
:returns: string representation of format
"""
raise NotImplementedError()
def __repr__(self):
return '<BaseFormatter> {}'.format(self.name)
pygeoapi/formatters/csv_.py
+98
View File
@@ -0,0 +1,98 @@
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
import io
import logging
import unicodecsv as csv
from pygeoapi.formatters.base import BaseFormatter
LOGGER = logging.getLogger(__name__)
class CSVFormatter(BaseFormatter):
"""CSV formatter"""
def __init__(self, geom=False):
"""
Initialize object
:param geom: whether to emit geometry (default False)
:returns: pygeoapi.formatters.csv_.CSVFormatter
"""
BaseFormatter.__init__(self, 'csv', geom)
self.mimetype = 'text/plain'
def write(self, options={}, data=None):
"""
Generate data in CSV format
:param options: CSV formatting options
:param data: dict of GeoJSON data
:returns: string representation of format
"""
is_point = False
try:
fields = list(data['features'][0]['properties'].keys())
except IndexError:
LOGGER.error('no features')
return str()
if self.geom:
LOGGER.debug('Including point geometry')
if data['features'][0]['geometry']['type'] == 'Point':
fields.insert(0, 'x')
fields.insert(1, 'y')
is_point = True
else:
# TODO: implement wkt geometry serialization
LOGGER.debug('not a point geometry, skipping')
LOGGER.debug('CSV fields: {}'.format(fields))
output = io.BytesIO()
writer = csv.DictWriter(output, fields)
writer.writeheader()
for feature in data['features']:
fp = feature['properties']
if is_point:
fp['x'] = feature['geometry']['coordinates'][0]
fp['y'] = feature['geometry']['coordinates'][1]
LOGGER.debug(fp)
writer.writerow(fp)
return output.getvalue()
def __repr__(self):
return '<CSVFormatter> {}'.format(self.mimetype)
requirements.txt
+1
View File
@@ -1,3 +1,4 @@
click
Flask
PyYAML
unicodecsv
tests/test_csv_formatter.py
+80
View File
@@ -0,0 +1,80 @@
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
import csv
import io
import pytest
from pygeoapi.formatters.csv_ import CSVFormatter
@pytest.fixture()
def fixture():
data = {
'features': [{
'geometry': {
'type': 'Point',
'coordinates': [
-130.44472222222223,
54.28611111111111
]
},
'type': 'Feature',
'properties': {
'id': 1972,
'foo': 'bar',
'title': None,
},
'ID': 48693
}]
}
return data
def test_csv_formatter(fixture):
f = CSVFormatter(geom=True)
f_csv = f.write(data=fixture)
buffer = io.StringIO(f_csv.decode('utf-8'))
reader = csv.DictReader(buffer)
header = list(reader.fieldnames)
assert len(header) == 5
assert 'x' in header
assert 'y' in header
data = next(reader)
assert data['x'] == '-130.44472222222223'
assert data['y'] == '54.28611111111111'
assert data['id'] == '1972'
assert data['foo'] == 'bar'
assert data['title'] == ''