Spatiallite3 provider implementation (#17)

* 	new file:   docker/Dockerfile
Dockerfile for pygeoapi

* 	modified:   pygeoapi/provider/__init__.py
	new file:   tests/data/ne_110m_lakes.sqlite
	new file:   tests/json_marshmallow.py
	new file:   tests/test_sqlite_provider.py

Sqlit implementation and testing marshmallows

* 	new file:   provider/sqlite.py
	new file:   provider/tmp_parser.py
Sqlite provider

* Testing sqlalchemy

* query for sqlite3

* Countries dataset, message in assert

* yml config

* table in data link and query implemented without limit

* PR of refactor

* functional sqlite3 driver

* flake8

* pipreq for complete list of requirements

* updated readme with working examples, extra requirements

* typos, SQLite removed Dockerfile and ne_100m_lakes.sqlite

* update requirements

* pypandoc in requirements-dev.txt

README.md

@@ -19,7 +19,7 @@ vi local.config.yml
 export PYGEOAPI_CONFIG=/path/to/local.config.yml
 # generate OpenAPI Document
 pygeoapi generate_openapi_document -c local.config.yml > openapi.yml
-export PYGEOAPI_SWAGGER=/path/to/openapi.yml
+export PYGEOAPI_OPENAPI=/path/to/openapi.yml
 pygeoapi serve
 ```
 
@@ -33,11 +33,16 @@ or
 # feature collection metadata
 curl http://localhost:5000/
 # conformance
-curl http://localhost:5000/api/conformance
+curl http://localhost:5000/conformance
 # feature collection
-curl http://localhost:5000/obs
+curl http://localhost:5000/collections/countries
+# feature collection limit 100
+curl http://localhost:5000/collections/countries/items?limit=100
 # feature
-curl http://localhost:5000/obs/371
+curl http://localhost:5000/collections/countries/items/1
+# nummer of hits
+curl http://localhost:5000/collections/countries/items?resulttype=hits
+
 ```
 
 ## Testing against Swagger UI

pygeoapi-config.yml

@@ -126,3 +126,33 @@ datasets:
             name: GeoJSON
             data: tests/data/ne_110m_lakes.geojson
             id_field: null  # null indicates use feature enumeration
+     
+    countries:
+        type: Polygon
+        id_field: ogc_fid  # null indicates use feature enumeration
+        title: Countries in the world
+        description: Countries of the world
+        keywords:
+            - countries
+            - natural eart
+        crs:
+            - CRS84
+        links:
+            - type: text/html
+              description: information
+              url: http://www.naturalearthdata.com/
+            - type: text/html
+              description: download
+              url: http://www.naturalearthdata.com/
+        extents:
+            spatial:
+                bbox: [-180,-90,180,90]
+            temporal:
+                begin: None
+                end: now  # or empty
+        provider: 
+            name: SQlite
+            data: tests/data/ne_110m_admin_0_countries.sqlite
+            id_field: ogc_fid
+            table: ne_110m_admin_0_countries
+            

pygeoapi/provider/__init__.py

@@ -35,7 +35,9 @@ LOGGER = logging.getLogger(__name__)
 PROVIDERS = {
     'CSV': 'pygeoapi.provider.csv_.CSVProvider',
     'Elasticsearch': 'pygeoapi.provider.elasticsearch_.ElasticsearchProvider',
-    'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider'
+    'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider',
+    'SQlite': 'pygeoapi.provider.sqlite.SQLiteProvider',
+
 }
 
 
@@ -69,5 +71,4 @@ def load_provider(provider_def):
 
 class InvalidProviderError(Exception):
     """invalid provider"""
-
     pass

pygeoapi/provider/sqlite.py

@@ -0,0 +1,202 @@
+# =================================================================
+#
+# Authors: Jorge Samuel Mendes de Jesus <jorge.dejesus@geocat.net>
+#
+# Copyright (c) 2018 Jorge Samuel Mendes de Jesus
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+#
+
+import sqlite3
+import logging
+import os
+import geojson
+from pygeoapi.provider.base import BaseProvider
+from pygeoapi.provider import InvalidProviderError
+
+LOGGER = logging.getLogger(__name__)
+
+
+class SQLiteProvider(object):
+    """Generic provide for SQLITE using sqlite3 module.
+    This module requires install of libsqlite3-mod-spatialite
+    TODO: DELETE, UPDATE, CREATE
+    """
+
+    def __init__(self, provider_def):
+        """
+        SQLiteProvider Class constructor
+
+        :param privider_def: provider definitions from yml pygeoapi-config.
+        data,id_field, name set in parent class
+
+        :returns: pygeoapi.providers.base.SQLiteProvider
+        """
+        BaseProvider.__init__(self, provider_def)
+
+        self.table = provider_def['table']
+
+        self.dataDB = None
+
+        LOGGER.debug('Setting Sqlite propreties:')
+        LOGGER.debug('Data source:{}'.format(self.data))
+        LOGGER.debug('Name:{}'.format(self.name))
+        LOGGER.debug('ID_field:{}'.format(self.id_field))
+        LOGGER.debug('Table:{}'.format(self.table))
+
+    def __response_feature_collection(self):
+        """Assembles GeoJSON output from DB query
+
+        :returns: GeoJSON FeaturesCollection
+        """
+
+        feature_list = list()
+        for row_data in self.dataDB:
+            row_data = dict(row_data)  # sqlite3.Row is doesnt support pop
+            geom = geojson.loads(row_data['AsGeoJSON(geometry)'])
+            del row_data['AsGeoJSON(geometry)']
+            feature = geojson.Feature(geometry=geom, properties=row_data)
+            feature_list.append(feature)
+
+        feature_collection = geojson.FeatureCollection(feature_list)
+
+        return feature_collection
+
+    def __response_feature_hits(self, hits):
+        """Assembles GeoJSON/Feature number
+
+        :returns: GeoJSON FeaturesCollection
+        """
+
+        feature_collection = geojson.FeatureCollection([])
+        feature_collection['numberMatched'] = str(hits)
+        return feature_collection
+
+    def __load(self):
+        """
+        Private method for loading spatiallite,
+        get the table structure and dump geometry
+
+        :returns: sqlite3.Cursor
+        """
+
+        if (os.path.exists(self.data)):
+            conn = sqlite3.connect(self.data)
+        else:
+            raise InvalidProviderError
+
+        conn.row_factory = sqlite3.Row
+        cursor = conn.cursor()
+        cursor.execute("SELECT load_extension('mod_spatialite')")
+        cursor.execute("PRAGMA table_info({})".format(self.table))
+
+        result = cursor.fetchall()
+        try:
+            # TODO: Better exceptions declaring
+            # InvalidProviderError as Parent class
+            assert len(result), "Table not found"
+            assert len([item for item in result
+                        if item['pk'] == 1]), "Primary key not found"
+            assert len([item for item in result
+                        if self.id_field in item]), "id_field not present"
+            assert len([item for item in result
+                        if 'GEOMETRY' in item]), "GEOMETRY column not found"
+
+        except InvalidProviderError:
+            raise
+
+        self.columns = [item[1] for item in result if item[1] != 'GEOMETRY']
+        self.columns = ",".join(self.columns)+",AsGeoJSON(geometry)"
+
+        return cursor
+
+    def query(self, startindex=0, limit=10, resulttype='results'):
+        """
+        Query Sqlite for all the content.
+        e,g: http://localhost:5000/collections/countries/items?
+        limit=1&resulttype=results
+
+        :param startindex: starting record to return (default 0)
+        :param limit: number of records to return (default 10)
+        :param resulttype: return results or hit limit (default results)
+
+        :returns: GeoJSON FeaturesCollection
+        """
+        LOGGER.debug('Querying Sqlite')
+
+        cursor = self.__load()
+
+        LOGGER.debug('Got cursor from DB')
+
+        if resulttype == 'hits':
+            res = cursor.execute("select count(*) as hits from {};".format(
+                self.table))
+
+            hits = res.fetchone()["hits"]
+            return self.__response_feature_hits(hits)
+
+        end_index = startindex + limit
+        # Not working
+        # http://localhost:5000/collections/countries/items/?startindex=10
+        sql_query = "select {} from {} where rowid >= ? \
+        and rowid <= ?;".format(self.columns, self.table)
+
+        LOGGER.debug('SQL Query:{}'.format(sql_query))
+        LOGGER.debug('Start Index:{}'.format(startindex))
+        LOGGER.debug('End Index'.format(end_index))
+
+        self.dataDB = cursor.execute(sql_query, (startindex, end_index, ))
+
+        feature_collection = self.__response_feature_collection()
+        return feature_collection
+
+    def get(self, identifier):
+        """
+        Query the provider for a specific
+        feature id e.g: /collections/countries/items/1
+
+        :param identifier: feature id
+
+        :returns: GeoJSON FeaturesCollection
+        """
+
+        LOGGER.debug('Get item from Sqlite')
+
+        cursor = self.__load()
+
+        LOGGER.debug('Got cursor from DB')
+
+        sql_query = "select {} from {} where {}==?;".format(self.columns,
+                                                            self.table,
+                                                            self.id_field)
+
+        LOGGER.debug('SQL Query:{}'.format(sql_query))
+        LOGGER.debug('Identifier:{}'.format(identifier))
+
+        self.dataDB = cursor.execute(sql_query, (identifier, ))
+
+        feature_collection = self.__response_feature_collection()
+        return feature_collection
+
+    def __repr__(self):
+        return '<SQliteProvider> {},{}'.format(self.data, self.table)

requirements-dev.txt

@@ -1,12 +1,12 @@
-coverage
-docutils
-flake8
-pypandoc
-twine
-wheel
-
+coverage==4.5.1
+docutils==0.14
+flake8==3.5.0
+twine==1.11.0
+wheel==0.31.0
+pypandoc==1.4
+pytest==3.5.0
 # the packages below are used to handle HTTP/SSL
 # errors when uploading to PyPI (https://github.com/pypa/twine/issues/273)
-pyOpenSSL
-ndg-httpsclient
-pyasn1
+pyOpenSSL==17.5.0
+ndg-httpsclient==0.4.4
+pyasn1==0.4.2

requirements.txt

@@ -1,7 +1,7 @@
-click
-elasticsearch
-flask
-flask_cors
-pyyaml
-requests
-shapely
+elasticsearch==6.2.0
+click==6.7
+Flask==0.12.2
+Shapely==1.5.7
+Flask_Cors==3.0.3
+PyYAML==3.12
+geojson==2.3.0

tests/test_sqlite_provider.py

@@ -0,0 +1,37 @@
+# Needs to be run like: pytest -s test_sqlite_provider.py
+# In eclipse we need to set PYGEOAPI_CONFIG, Run>Debug Configurations>
+# (Arguments as py.test and set external variables to the correct config path)
+
+import pytest
+from pygeoapi.provider.sqlite import SQLiteProvider
+
+
+@pytest.fixture()
+def config():
+    return {
+            'name': 'Sqlite',
+            'data': './tests/data/ne_110m_admin_0_countries.sqlite',
+            'id_field': "ogc_fid",
+            'table': 'ne_110m_admin_0_countries'}
+
+
+def test_query(config):
+    """Testing query for a valid JSON object with geometry"""
+
+    p = SQLiteProvider(config)
+    feature_collection = p.query()
+    assert feature_collection.get('type', None) == "FeatureCollection"
+    features = feature_collection.get('features', None)
+    assert features is not None
+    feature = features[0]
+    properties = feature.get("properties", None)
+    assert properties is not None
+    geometry = feature.get("geometry", None)
+    assert geometry is not None
+
+
+def test_get(config):
+    p = SQLiteProvider(config)
+    results = p.get(118)
+    assert len(results['features']) == 1
+    assert "Netherlands" in results['features'][0]['properties']['admin']