implement logging, first pass cli (#14)

This commit is contained in:
Tom Kralidis
2018-03-08 08:38:42 -05:00
committed by GitHub
parent 957df9b844
commit d6e4b1dc00
15 changed files with 474 additions and 76 deletions
+4
View File
@@ -99,3 +99,7 @@ ENV/
# mypy
.mypy_cache/
# pygeoapi artifacts
local.config.yml
local.swagger.yml
+11 -5
View File
@@ -15,12 +15,15 @@ pip3 install -r requirements-dev.txt
pip3 install -e .
cp openapi/wfs/0.0.1/pygeoapi-openapi.yml local.swagger.yml
cp pygeoapi-config.yml local.config.yml
python flask_app.py
vi local.config.yml
# TODO: what is most important to edit?
vi local.swagger.yml
# TODO: what is most important to edit?
export PYGEOAPI_CONFIG=/path/to/local.config.yml
export PYGEOAPI_SWAGGER=/path/to/local.swagger.yml
pygeoapi serve
```
Edit `local.config.yml` and `local.swagger.yml`
## Example requests
Try the swagger ui at `http://localhost:5000/ui`
@@ -30,7 +33,10 @@ or
```bash
# feature collection metadata
curl http://localhost:5000/
curl http://localhost:5000/api
# conformance
curl http://localhost:5000/api/conformance
# feature collection
curl http://localhost:5000/obs
# feature
curl http://localhost:5000/obs/371
```
+5 -4
View File
@@ -22,6 +22,7 @@
# OR OTHER DEALINGS IN THE SOFTWARE.
#
##############################################################################
swagger: '2.0'
info:
title: A sample API conforming to the OGC Web Feature Service standard
@@ -93,11 +94,11 @@ paths:
summary: retrieve observation features
description: >-
Sample observations
operationId: pygeoapi.views.getFeatures
operationId: pygeoapi.views.get_features
tags:
- Features
parameters:
- name: ds # constant parameter
- name: dataset # constant parameter
in: query
type: string
enum:
@@ -125,11 +126,11 @@ paths:
'/obs/{id}':
get:
summary: retrieve a feature
operationId: pygeoapi.views.getFeature
operationId: pygeoapi.views.get_feature
tags:
- Features
parameters:
- name: ds # constant parameter
- name: dataset # constant parameter
in: query
type: string
enum:
+89 -1
View File
@@ -1,6 +1,47 @@
server:
host: localhost
port: 5000
mimetype: application/json; charset=UTF-8
encoding: utf-8
language: en-US
cors: true
pretty_print: true
limit: 10
logging:
level: INFO
#logfile: /tmp/pygeoapi.log
metadata:
identification:
title: pygeoapi
abstract: pygeoapi provides an API to geospatial data
keywords:
- geospatial
- data
- api
keywords_type: theme
fees: None
accessconstraints: None
provider:
name: Organization Name
url: https://github.com/geopython/pygeoapi
contact:
name: Lastname, Firstname
position: Position Title
address: Mailing Address
city: City
stateorprovince: Administrative Area
postalcode: Zip or Postal Code
country: Country
phone: +xx-xxx-xxx-xxxx
fax: +xx-xxx-xxx-xxxx
email: you@example.org
url: Contact URL
hours: Hours of Service
instructions: During hours of service. Off on weekends.
role: pointOfContact
datasets:
obs:
type: Point
@@ -24,5 +65,52 @@ datasets:
end: 2007-10-30T08:57:29Z
data:
type: CSV
url: ./tests/data/obs.csv
url: file:///tests/data/obs.csv
id_field: id
landsat-aws:
type: Polygon
title: my dataset
abstract: my dataset
keywords:
- kw1
- kw2
crs:
- CRS84
links:
- type: information
url: http://example.org/dataset/index.html
- type: download
url: http://example.org/dataset/data.tgz
extents:
spatial:
bbox: [-180,-90,180,90]
temporal:
begin: 2011-11-11
end: now # or empty
data:
type: Elasticsearch
url: http://localhost:9200/index/type
id_field: id_
lakes:
type: Polygon
title: Large Lakes
abstract: lakes of the world, public domain
keywords:
- lakes
crs:
- CRS84
links:
- type: information
url: http://www.naturalearthdata.com/
- type: download
url: http://www.naturalearthdata.com/
extents:
spatial:
bbox: [-180,-90,180,90]
temporal:
begin: 2011-11-11
end: now # or empty
data:
type: GeoJSON
url: file:///tests/data/ne_110m_lakes.geojson
id_field: null # null indicates use feature enumeration
+5
View File
@@ -29,6 +29,8 @@
import click
from pygeoapi.flask_app import serve
__version__ = '0.1.dev0'
@@ -36,3 +38,6 @@ __version__ = '0.1.dev0'
@click.version_option(version=__version__)
def cli():
pass
cli.add_command(serve)
+36 -2
View File
@@ -1,4 +1,38 @@
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
# Norman Barker <norman.barker@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
import os
import yaml
with open('local.config.yml', 'r') as ymlfile:
settings = yaml.load(ymlfile)
with open(os.environ.get('PYGEOAPI_CONFIG')) as ff:
settings = yaml.load(ff)
settings['swagger'] = os.environ.get('PYGEOAPI_SWAGGER')
+30 -4
View File
@@ -1,7 +1,7 @@
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
# : Norman Barker <norman.barker@gmail.com>
# Norman Barker <norman.barker@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
#
@@ -28,12 +28,38 @@
#
# =================================================================
import click
import connexion
from pygeoapi.log import setup_logger
from pygeoapi.config import settings
if __name__ == '__main__':
app = connexion.FlaskApp(__name__, port=int(settings['server']['port']), specification_dir='.')
api = app.add_api('local.swagger.yml', debug=True, strict_validation=True, arguments={'host': f'{settings["server"]["host"]}:{settings["server"]["port"]}'})
@click.command()
@click.pass_context
@click.option('--host', '-h', default='localhost', help='Hostname')
@click.option('--port', '-p', default=5000, help='port')
@click.option('--debug', '-d', default=False, is_flag=True, help='debug')
def serve(ctx, host, port, debug=False):
"""Serve pygeoapi via Flask"""
if port is not None:
port_ = port
else:
port_ = settings['server']['port']
if host is not None:
host_ = host
else:
host_ = settings['server']['host']
app = connexion.FlaskApp(__name__, port=port_, specification_dir='.')
hostport = '{}:{}'.format(host_, port_)
api = app.add_api(settings['swagger'], debug=debug, strict_validation=True,
arguments={'host': hostport})
settings['api'] = api.specification
setup_logger()
app.run()
+72
View File
@@ -0,0 +1,72 @@
# =================================================================
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
import logging
import sys
from pygeoapi.config import settings
LOGGER = logging.getLogger(__name__)
def setup_logger():
"""
Setup configuration
:returns: void (creates logging instance)
"""
log_format = \
'[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s'
loglevels = {
'CRITICAL': logging.CRITICAL,
'ERROR': logging.ERROR,
'WARNING': logging.WARNING,
'INFO': logging.INFO,
'DEBUG': logging.DEBUG,
'NOTSET': logging.NOTSET,
}
formatter = logging.Formatter(log_format)
log_handler = logging.NullHandler()
if 'level' in settings['logging']:
loglevel = loglevels[settings['logging']['level']]
log_handler = logging.StreamHandler(sys.stdout)
if 'logfile' in settings['logging']:
log_handler = logging.FileHandler(settings['logging']['logfile'])
log_handler.setLevel(loglevel)
log_handler.setFormatter(formatter)
LOGGER.addHandler(log_handler)
LOGGER.debug('Logging initialized')
return
+9 -1
View File
@@ -28,6 +28,9 @@
# =================================================================
import importlib
import logging
LOGGER = logging.getLogger(__name__)
PROVIDERS = {
'CSV': 'pygeoapi.provider.csv.CSVProvider',
@@ -45,11 +48,16 @@ def load_provider(provider_obj):
:returns: provider object
"""
LOGGER.debug('Providers: {}'.format(PROVIDERS))
provider_name = provider_obj['type']
if provider_name not in PROVIDERS.keys():
raise InvalidProviderError('provider not found')
msg = 'Provider {} not found'.format(provider_name)
LOGGER.exception(msg)
raise InvalidProviderError(msg)
packagename, classname = PROVIDERS[provider_name].rsplit('.', 1)
LOGGER.debug('package name: {}'.format(packagename))
LOGGER.debug('class name: {}'.format(classname))
module = importlib.import_module(packagename)
class_ = getattr(module, classname)
+13 -1
View File
@@ -27,23 +27,35 @@
#
# =================================================================
import logging
from urllib.request import urlparse
LOGGER = logging.getLogger(__name__)
class BaseProvider(object):
"""generic Provider ABC"""
def __init__(self, definition):
"""initializer"""
"""
Initialize object
:param definition: dict of dataset data definition
:returns: pygeoapi.providers.base.BaseProvider
"""
self.type = definition['type']
self.id_field = definition['id_field']
parsed_url = urlparse(definition['url'])
LOGGER.debug('Parsed URL: {}'.format(parsed_url))
if parsed_url.scheme == 'file':
LOGGER.debug('URL is a file')
self.url = parsed_url.path
else:
LOGGER.debug('URL is really a URL')
self.url = definition['url']
def query(self):
+39 -16
View File
@@ -29,65 +29,88 @@
import csv
import itertools
import logging
from shapely import wkt
from shapely.geometry import mapping
from pygeoapi.provider.base import BaseProvider
LOGGER = logging.getLogger(__name__)
class CSVProvider(BaseProvider):
"""CSV provider"""
def __init__(self, definition):
"""initializer"""
"""
Initialize object
:param definition: dict of dataset data definition
:returns: pygeoapi.providers.csv.CSVProvider
"""
BaseProvider.__init__(self, definition)
with open(self.url) as ff:
self.data = csv.DictReader(ff)
def _load(self, startindex=0, count=10, resulttype='results',
identifier=None):
"""
Load CSV data
:param startindex: starting record to return (default 0)
:param count: number of records to return (default 10)
:param resulttype: return results or hit count (default results)
:returns: dict of GeoJSON FeatureCollection
"""
def _load(self, identifier=None):
feature_collection = {
'type': 'FeatureCollection',
'features': []
}
with open(self.url) as ff:
LOGGER.debug('Serializing DictReader')
data = csv.DictReader(ff)
feat = None
for row in data:
if resulttype == 'hits':
LOGGER('Retrurning hits only')
feature_collection['numberMatched'] = len(list(data))
return feature_collection
LOGGER.debug('Slicing CSV rows')
for row in itertools.islice(data, startindex, startindex+count):
feature = {'type': 'Feature'}
feature['ID'] = row.pop('id')
feature['geometry'] = mapping(wkt.loads(row.pop('geom')))
feature['properties'] = row
if identifier is not None and feature['ID'] == identifier:
feat = feature
break
return feature
feature_collection['features'].append(feature)
if identifier is not None:
return feat
else:
return feature_collection
return feature_collection
def query(self, startindex=0, count=10, resulttype='results'):
"""
CSV query
:returns: dict of 0..n GeoJSON features
:param startindex: starting record to return (default 0)
:param count: number of records to return (default 10)
:param resulttype: return results or hit count (default results)
:returns: dict of GeoJSON FeatureCollection
"""
return self._load()
return self._load(startindex, count, resulttype)
def get(self, identifier):
"""
query CSV id
:param identifier: feature id
:returns: dict of single GeoJSON feature
"""
return self._load(identifier)
return self._load(identifier=identifier)
def __repr__(self):
return '<CSVProvider> {}'.format(self.url)
+32 -3
View File
@@ -27,30 +27,49 @@
#
# =================================================================
import logging
from elasticsearch import Elasticsearch
from pygeoapi.provider.base import BaseProvider
LOGGER = logging.getLogger(__name__)
class ElasticsearchProvider(BaseProvider):
"""Elasticsearch Provider"""
def __init__(self, definition):
"""initializer"""
"""
Initialize object
:param definition: dict of dataset data definition
:returns: pygeoapi.providers.elasticsearch.ElasticsearchProvider
"""
BaseProvider.__init__(self, definition)
url_tokens = self.url.split('/')
LOGGER.debug('Setting Elasticsearch properties')
self.es_host = url_tokens[2]
self.index_name = url_tokens[-2]
self.type_name = url_tokens[-1]
self.es_host = url_tokens[2]
LOGGER.debug('host: {}'.format(self.es_host))
LOGGER.debug('index: {}'.format(self.index_name))
LOGGER.debug('type: {}'.format(self.type_name))
LOGGER.debug('Connecting to Elasticsearch')
self.es = Elasticsearch(self.es_host)
def query(self, startindex=0, count=10, resulttype='results'):
"""
query ES
query Elasticsearch index
:param startindex: starting record to return (default 0)
:param count: number of records to return (default 10)
:param resulttype: return results or hit count (default results)
:returns: dict of 0..n GeoJSON features
"""
@@ -60,14 +79,20 @@ class ElasticsearchProvider(BaseProvider):
'features': []
}
LOGGER.debug('Querying Elasticsearch')
if resulttype == 'hits':
LOGGER.debug('hits only specified')
count = 0
results = self.es.search(index=self.index_name, from_=startindex,
size=count)
if resulttype == 'hits':
feature_collection['numberMatched'] = results['hits']['total']
return feature_collection
LOGGER.debug('serializing features')
for feature in results['hits']['hits']:
id_ = feature['_source']['properties']['identifier']
LOGGER.debug('serializing id {}'.format(id_))
feature['_source']['ID'] = id_
feature_collection['features'].append(feature['_source'])
@@ -78,15 +103,19 @@ class ElasticsearchProvider(BaseProvider):
Get ES document by id
:param identifier: feature id
:returns: dict of single GeoJSON feature
"""
try:
LOGGER.debug('Fetching identifier {}'.format(identifier))
result = self.es.get(self.index_name, doc_type=self.type_name,
id=identifier)
LOGGER.debug('Serializing feature')
id_ = result['_source']['properties']['identifier']
result['_source']['ID'] = id_
except Exception as err:
LOGGER.error(err)
return None
return result['_source']
+127 -36
View File
@@ -28,57 +28,148 @@
#
# =================================================================
import logging
from flask import request
# from flask import request, url_for
from pygeoapi.config import settings
from pygeoapi.provider import load_provider
LOGGER = logging.getLogger(__name__)
def describe_collections(f='json'):
"""
Provide feature collection metadata
:param f: response format (default JSON)
:returns: dict of feature collection metadata
"""
# TODO allow other file return formats
if f.upper() == 'JSON':
fcm = {
'collections': []
}
if f.upper() != 'JSON':
msg = 'Unsupported format: {}'.format(f)
LOGGER.error(msg)
return msg, 400
for k, v in settings['datasets'].items():
collection = {'links': [], 'crs': []}
collection['collectionId'] = k
collection['title'] = v['title']
collection['description'] = v['abstract']
for crs in v['crs']:
collection['crs'].append(
'http://www.opengis.net/def/crs/OGC/1.3/{}'.format(crs))
collection['extent'] = v['extents']['spatial']['bbox']
fcm = {
'links': [],
'collections': []
}
for link in v['links']:
lnk = {'rel': link['type'], 'href': link['url']}
collection['links'].append(lnk)
url = '{}://{}'.format(request.scheme, settings['server']['host'])
if settings['server']['port'] not in [80, 443]:
url = '{}:{}'.format(url, settings['server']['port'])
fcm['collections'].append(collection)
# LOGGER.debug('Creating links')
# fcm['links'] = [{
# 'rel': 'self',
# 'type': 'application/json',
# 'title': 'this document',
# 'href': '{}{}'.format(url, url_for('index_json')),
# }, {
# 'rel': 'self',
# 'type': 'text/html',
# 'title': 'this document as HTML',
# 'href': '{}{}'.format(url, url_for('index_html')),
# }, {
# 'rel': 'self',
# 'type': 'application/openapi+json;version=3.0',
# 'title': 'the OpenAPI definition as JSON',
# 'href': '{}{}'.format(url, url_for('api_json')),
# }, {
# 'rel': 'self',
# 'type': 'text/html',
# 'title': 'the OpenAPI definition as HTML',
# 'href': '{}{}'.format(url, url_for('api_html')),
# }
# ]
LOGGER.debug('Creating collections')
for k, v in settings['datasets'].items():
collection = {'links': [], 'crs': []}
collection['name'] = k
collection['title'] = v['title']
collection['description'] = v['abstract']
for crs in v['crs']:
collection['crs'].append(
'http://www.opengis.net/def/crs/OGC/1.3/{}'.format(crs))
collection['extent'] = v['extents']['spatial']['bbox']
for link in v['links']:
lnk = {'rel': link['type'], 'href': link['url']}
collection['links'].append(lnk)
fcm['collections'].append(collection)
return fcm
return {'collections' : collection}
else:
return f'"{f}" not supported as a query parameter.', 400
def get_specification(f='json'):
if f.upper() == 'JSON':
return settings['api']
else:
return f'"{f}" not supported as a query parameter.', 400
return '{} not supported as a query parameter'.format(f), 400
def getFeatures(ds, start_index, count, result_type, bbox=None, f='json'):
if ds not in settings['datasets'].keys():
return f'dataset {ds} not found.', 400
else:
p = load_provider(settings['datasets'][ds]['data'])
return p.query()
def getFeature(ds, id, f='json'):
if ds not in settings['datasets'].keys():
return f'dataset {ds} not found.', 400
def get_features(dataset, startindex=0, count=10, resulttype='results',
bbox=None, f='json'):
"""
Queries feature collection
:param dataset: dataset to query
:param startindex: starting record to return (default 0)
:param count: number of records to return (default 10)
:param resulttype: return results or hit count (default results)
:param bbox: list of minx,miny,maxx,maxy
:param f: responase format (default GeoJSON)
:returns: dict of GeoJSON FeatureCollection
"""
if dataset not in settings['datasets'].keys():
msg = 'dataset {} not found'.format(dataset)
LOGGER.error(msg)
return msg, 400
else:
p = load_provider(settings['datasets'][ds]['data'])
feat = p.get(id)
if feat is None:
return f'feature "{id}" not found', 404
else:
return feat
LOGGER.debug('Loading provider')
p = load_provider(settings['datasets'][dataset]['data'])
LOGGER.debug('Querying provider')
LOGGER.debug('startindex: {}'.format(startindex))
LOGGER.debug('count: {}'.format(count))
LOGGER.debug('resulttype: {}'.format(resulttype))
results = p.query(startindex=int(startindex), count=int(count),
resulttype=resulttype)
return results
def get_feature(dataset, id, f='json'):
"""
Get a single feature
:param dataset: dataset to query
:param id: feature identifier
:param f: responase format (default GeoJSON)
:returns: dict of GeoJSON Feature
"""
if dataset not in settings['datasets'].keys():
msg = 'dataset {} not found'.format(dataset)
LOGGER.error(msg)
return msg, 400
LOGGER.debug('Loading provider')
p = load_provider(settings['datasets'][dataset]['data'])
LOGGER.debug('Fetching id {}'.format(id))
feature = p.get(id)
if feature is None:
msg = 'feature {} not found'.format(id)
LOGGER.warning(msg)
return msg, 404
return feature
+2 -2
View File
@@ -1,8 +1,8 @@
connexion
click
connexion
elasticsearch
flask
flask_cors
pyyaml
requests
shapely
pyyaml
-1
View File
@@ -114,7 +114,6 @@ def test_update_safe_id(fixture, config):
assert 'Null' in results['features'][0]['properties']['name']
"""
def __init__(self, definition):
BaseProvider.__init__(self, definition)