diff --git a/README.md b/README.md index 2648fee..8bfdbe6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ vi local.config.yml export PYGEOAPI_CONFIG=/path/to/local.config.yml # generate OpenAPI Document pygeoapi generate_openapi_document -c local.config.yml > openapi.yml -export PYGEOAPI_SWAGGER=/path/to/openapi.yml +export PYGEOAPI_OPENAPI=/path/to/openapi.yml pygeoapi serve ``` @@ -33,11 +33,16 @@ or # feature collection metadata curl http://localhost:5000/ # conformance -curl http://localhost:5000/api/conformance +curl http://localhost:5000/conformance # feature collection -curl http://localhost:5000/obs +curl http://localhost:5000/collections/countries +# feature collection limit 100 +curl http://localhost:5000/collections/countries/items?limit=100 # feature -curl http://localhost:5000/obs/371 +curl http://localhost:5000/collections/countries/items/1 +# nummer of hits +curl http://localhost:5000/collections/countries/items?resulttype=hits + ``` ## Testing against Swagger UI diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml index ce1e72c..d944edb 100644 --- a/pygeoapi-config.yml +++ b/pygeoapi-config.yml @@ -126,3 +126,33 @@ datasets: name: GeoJSON data: tests/data/ne_110m_lakes.geojson id_field: null # null indicates use feature enumeration + + countries: + type: Polygon + id_field: ogc_fid # null indicates use feature enumeration + title: Countries in the world + description: Countries of the world + keywords: + - countries + - natural eart + crs: + - CRS84 + links: + - type: text/html + description: information + url: http://www.naturalearthdata.com/ + - type: text/html + description: download + url: http://www.naturalearthdata.com/ + extents: + spatial: + bbox: [-180,-90,180,90] + temporal: + begin: None + end: now # or empty + provider: + name: SQlite + data: tests/data/ne_110m_admin_0_countries.sqlite + id_field: ogc_fid + table: ne_110m_admin_0_countries + diff --git a/pygeoapi/provider/__init__.py b/pygeoapi/provider/__init__.py index 1fd5edc..a8e7d10 100644 --- a/pygeoapi/provider/__init__.py +++ b/pygeoapi/provider/__init__.py @@ -35,7 +35,9 @@ LOGGER = logging.getLogger(__name__) PROVIDERS = { 'CSV': 'pygeoapi.provider.csv_.CSVProvider', 'Elasticsearch': 'pygeoapi.provider.elasticsearch_.ElasticsearchProvider', - 'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider' + 'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider', + 'SQlite': 'pygeoapi.provider.sqlite.SQLiteProvider', + } @@ -69,5 +71,4 @@ def load_provider(provider_def): class InvalidProviderError(Exception): """invalid provider""" - pass diff --git a/pygeoapi/provider/sqlite.py b/pygeoapi/provider/sqlite.py new file mode 100644 index 0000000..e9174d5 --- /dev/null +++ b/pygeoapi/provider/sqlite.py @@ -0,0 +1,202 @@ +# ================================================================= +# +# Authors: Jorge Samuel Mendes de Jesus +# +# Copyright (c) 2018 Jorge Samuel Mendes de Jesus +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# + +import sqlite3 +import logging +import os +import geojson +from pygeoapi.provider.base import BaseProvider +from pygeoapi.provider import InvalidProviderError + +LOGGER = logging.getLogger(__name__) + + +class SQLiteProvider(object): + """Generic provide for SQLITE using sqlite3 module. + This module requires install of libsqlite3-mod-spatialite + TODO: DELETE, UPDATE, CREATE + """ + + def __init__(self, provider_def): + """ + SQLiteProvider Class constructor + + :param privider_def: provider definitions from yml pygeoapi-config. + data,id_field, name set in parent class + + :returns: pygeoapi.providers.base.SQLiteProvider + """ + BaseProvider.__init__(self, provider_def) + + self.table = provider_def['table'] + + self.dataDB = None + + LOGGER.debug('Setting Sqlite propreties:') + LOGGER.debug('Data source:{}'.format(self.data)) + LOGGER.debug('Name:{}'.format(self.name)) + LOGGER.debug('ID_field:{}'.format(self.id_field)) + LOGGER.debug('Table:{}'.format(self.table)) + + def __response_feature_collection(self): + """Assembles GeoJSON output from DB query + + :returns: GeoJSON FeaturesCollection + """ + + feature_list = list() + for row_data in self.dataDB: + row_data = dict(row_data) # sqlite3.Row is doesnt support pop + geom = geojson.loads(row_data['AsGeoJSON(geometry)']) + del row_data['AsGeoJSON(geometry)'] + feature = geojson.Feature(geometry=geom, properties=row_data) + feature_list.append(feature) + + feature_collection = geojson.FeatureCollection(feature_list) + + return feature_collection + + def __response_feature_hits(self, hits): + """Assembles GeoJSON/Feature number + + :returns: GeoJSON FeaturesCollection + """ + + feature_collection = geojson.FeatureCollection([]) + feature_collection['numberMatched'] = str(hits) + return feature_collection + + def __load(self): + """ + Private method for loading spatiallite, + get the table structure and dump geometry + + :returns: sqlite3.Cursor + """ + + if (os.path.exists(self.data)): + conn = sqlite3.connect(self.data) + else: + raise InvalidProviderError + + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + cursor.execute("SELECT load_extension('mod_spatialite')") + cursor.execute("PRAGMA table_info({})".format(self.table)) + + result = cursor.fetchall() + try: + # TODO: Better exceptions declaring + # InvalidProviderError as Parent class + assert len(result), "Table not found" + assert len([item for item in result + if item['pk'] == 1]), "Primary key not found" + assert len([item for item in result + if self.id_field in item]), "id_field not present" + assert len([item for item in result + if 'GEOMETRY' in item]), "GEOMETRY column not found" + + except InvalidProviderError: + raise + + self.columns = [item[1] for item in result if item[1] != 'GEOMETRY'] + self.columns = ",".join(self.columns)+",AsGeoJSON(geometry)" + + return cursor + + def query(self, startindex=0, limit=10, resulttype='results'): + """ + Query Sqlite for all the content. + e,g: http://localhost:5000/collections/countries/items? + limit=1&resulttype=results + + :param startindex: starting record to return (default 0) + :param limit: number of records to return (default 10) + :param resulttype: return results or hit limit (default results) + + :returns: GeoJSON FeaturesCollection + """ + LOGGER.debug('Querying Sqlite') + + cursor = self.__load() + + LOGGER.debug('Got cursor from DB') + + if resulttype == 'hits': + res = cursor.execute("select count(*) as hits from {};".format( + self.table)) + + hits = res.fetchone()["hits"] + return self.__response_feature_hits(hits) + + end_index = startindex + limit + # Not working + # http://localhost:5000/collections/countries/items/?startindex=10 + sql_query = "select {} from {} where rowid >= ? \ + and rowid <= ?;".format(self.columns, self.table) + + LOGGER.debug('SQL Query:{}'.format(sql_query)) + LOGGER.debug('Start Index:{}'.format(startindex)) + LOGGER.debug('End Index'.format(end_index)) + + self.dataDB = cursor.execute(sql_query, (startindex, end_index, )) + + feature_collection = self.__response_feature_collection() + return feature_collection + + def get(self, identifier): + """ + Query the provider for a specific + feature id e.g: /collections/countries/items/1 + + :param identifier: feature id + + :returns: GeoJSON FeaturesCollection + """ + + LOGGER.debug('Get item from Sqlite') + + cursor = self.__load() + + LOGGER.debug('Got cursor from DB') + + sql_query = "select {} from {} where {}==?;".format(self.columns, + self.table, + self.id_field) + + LOGGER.debug('SQL Query:{}'.format(sql_query)) + LOGGER.debug('Identifier:{}'.format(identifier)) + + self.dataDB = cursor.execute(sql_query, (identifier, )) + + feature_collection = self.__response_feature_collection() + return feature_collection + + def __repr__(self): + return ' {},{}'.format(self.data, self.table) diff --git a/requirements-dev.txt b/requirements-dev.txt index a4d5c14..a71034a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,12 +1,12 @@ -coverage -docutils -flake8 -pypandoc -twine -wheel - +coverage==4.5.1 +docutils==0.14 +flake8==3.5.0 +twine==1.11.0 +wheel==0.31.0 +pypandoc==1.4 +pytest==3.5.0 # the packages below are used to handle HTTP/SSL # errors when uploading to PyPI (https://github.com/pypa/twine/issues/273) -pyOpenSSL -ndg-httpsclient -pyasn1 +pyOpenSSL==17.5.0 +ndg-httpsclient==0.4.4 +pyasn1==0.4.2 diff --git a/requirements.txt b/requirements.txt index 9896e14..6345885 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -click -elasticsearch -flask -flask_cors -pyyaml -requests -shapely +elasticsearch==6.2.0 +click==6.7 +Flask==0.12.2 +Shapely==1.5.7 +Flask_Cors==3.0.3 +PyYAML==3.12 +geojson==2.3.0 \ No newline at end of file diff --git a/tests/data/ne_110m_admin_0_countries.sqlite b/tests/data/ne_110m_admin_0_countries.sqlite new file mode 100644 index 0000000..d79b17a Binary files /dev/null and b/tests/data/ne_110m_admin_0_countries.sqlite differ diff --git a/tests/test_sqlite_provider.py b/tests/test_sqlite_provider.py new file mode 100644 index 0000000..4e0d5e5 --- /dev/null +++ b/tests/test_sqlite_provider.py @@ -0,0 +1,37 @@ +# Needs to be run like: pytest -s test_sqlite_provider.py +# In eclipse we need to set PYGEOAPI_CONFIG, Run>Debug Configurations> +# (Arguments as py.test and set external variables to the correct config path) + +import pytest +from pygeoapi.provider.sqlite import SQLiteProvider + + +@pytest.fixture() +def config(): + return { + 'name': 'Sqlite', + 'data': './tests/data/ne_110m_admin_0_countries.sqlite', + 'id_field': "ogc_fid", + 'table': 'ne_110m_admin_0_countries'} + + +def test_query(config): + """Testing query for a valid JSON object with geometry""" + + p = SQLiteProvider(config) + feature_collection = p.query() + assert feature_collection.get('type', None) == "FeatureCollection" + features = feature_collection.get('features', None) + assert features is not None + feature = features[0] + properties = feature.get("properties", None) + assert properties is not None + geometry = feature.get("geometry", None) + assert geometry is not None + + +def test_get(config): + p = SQLiteProvider(config) + results = p.get(118) + assert len(results['features']) == 1 + assert "Netherlands" in results['features'][0]['properties']['admin']