Initial version of mongo provider (#321)

2020-01-07 14:24:45 +02:00
parent c9abac21d4
commit 0fc8d9502a
6 changed files with 452 additions and 1 deletions
@@ -16,6 +16,7 @@ python:
 services:
  - elasticsearch
  - postgresql
+  - mongodb

 addons:
  postgresql: 9.6
@@ -27,6 +28,7 @@ before_install:
  - sudo apt-get install -y libsqlite3-mod-spatialite devscripts fakeroot debhelper
  - sudo apt-get install -y postgresql-9.6-postgis-2.4
  - sudo apt-get install -y libgdal-dev gdal-bin
+  - sudo apt-get install -y mongodb

 install:
  # follow GDAL installed version for Python bindings
@@ -39,6 +41,7 @@ install:
 before_script:
  - sleep 20
  - python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid
+  - python3 tests/load_mongo_data.py tests/data/ne_110m_populated_places_simple.geojson
  - pygeoapi generate-openapi-document -c pygeoapi-config.yml > pygeoapi-openapi.yml
  - psql -U postgres -c 'create database test'
  - psql -U postgres -d test -c 'create extension postgis'
@@ -43,7 +43,8 @@ PLUGINS = {
        'GeoPackage': 'pygeoapi.provider.geopackage.GeoPackageProvider',
        'OGR': 'pygeoapi.provider.ogr.OGRProvider',
        'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider',
-        'SQLiteGPKG': 'pygeoapi.provider.sqlite.SQLiteGPKGProvider'
+        'SQLiteGPKG': 'pygeoapi.provider.sqlite.SQLiteGPKGProvider',
+        'MongoDB': 'pygeoapi.provider.mongo.MongoProvider'
    },
    'formatter': {
        'CSV': 'pygeoapi.formatter.csv_.CSVFormatter'
@@ -0,0 +1,175 @@
+# =================================================================
+#
+# Authors: Timo Tuunanen <timo.tuunanen@rdvelho.com>
+#
+# Copyright (c) 2019 Timo Tuunanen
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+
+import logging
+
+from bson import Code
+from pymongo import MongoClient
+from pymongo import GEOSPHERE
+from pymongo import ASCENDING, DESCENDING
+from pymongo.collection import ObjectId
+from pygeoapi.provider.base import BaseProvider
+
+LOGGER = logging.getLogger(__name__)
+
+
+class MongoProvider(BaseProvider):
+    """Generic provider for Mongodb.
+    """
+
+    def __init__(self, provider_def):
+        """
+        MongoProvider Class constructor
+
+        :param provider_def: provider definitions from yml pygeoapi-config.
+                             data,id_field, name set in parent class
+
+        :returns: pygeoapi.providers.mongo.MongoProvider
+        """
+        # this is dummy value never used in case of Mongo.
+        # Mongo id field is _id
+        provider_def.setdefault('id_field', '_id')
+
+        BaseProvider.__init__(self, provider_def)
+
+        LOGGER.info('Mongo source config: {}'.format(self.data))
+
+        dbclient = MongoClient(self.data)
+        self.featuredb = dbclient.get_default_database()
+        self.collection = provider_def['collection']
+        self.featuredb[self.collection].create_index([("geometry", GEOSPHERE)])
+
+    def get_fields(self):
+        """
+        Get provider field information (names, types)
+
+        :returns: dict of fields
+        """
+        map = Code(
+            "function() { for (var key in this.properties) "
+            "{ emit(key, null); } }")
+        reduce = Code("function(key, stuff) { return null; }")
+        result = self.featuredb[self.collection].map_reduce(
+            map, reduce, "myresults")
+        return result.distinct('_id')
+
+    def _get_feature_list(self, filterObj, sortList=[], skip=0, maxitems=1):
+        featurecursor = self.featuredb[self.collection].find(filterObj)
+
+        if sortList:
+            featurecursor = featurecursor.sort(sortList)
+
+        matchCount = self.featuredb[self.collection].count_documents(filterObj)
+        featurecursor.skip(skip)
+        featurecursor.limit(maxitems)
+        featurelist = list(featurecursor)
+        for item in featurelist:
+            item['id'] = str(item.pop('_id'))
+
+        return featurelist, matchCount
+
+    def query(self, startindex=0, limit=10, resulttype='results',
+              bbox=[], datetime=None, properties=[], sortby=[]):
+        """
+        query the provider
+
+        :returns: dict of 0..n GeoJSON features
+        """
+        and_filter = []
+
+        if len(bbox) == 4:
+            x, y, w, h = map(float, bbox)
+            and_filter.append(
+                {'geometry': {'$geoWithin': {'$box': [[x, y], [w, h]]}}})
+
+        # This parameter is not working yet!
+        # gte is not sufficient to check date range
+        if datetime is not None:
+            assert isinstance(datetime.datetime, datetime)
+            and_filter.append({'properties.datetime': {'$gte': datetime}})
+
+        for prop in properties:
+            and_filter.append({"properties."+prop[0]: {'$eq': prop[1]}})
+
+        filterobj = {'$and': and_filter} if and_filter else {}
+
+        sort_list = [("properties." + sort['property'],
+                      ASCENDING if (sort['order'] == 'A') else DESCENDING)
+                     for sort in sortby]
+
+        featurelist, matchcount = self._get_feature_list(filterobj,
+                                                         sortList=sort_list,
+                                                         skip=startindex,
+                                                         maxitems=limit)
+
+        if resulttype == 'hits':
+            featurelist = []
+
+        feature_collection = {
+            'type': 'FeatureCollection',
+            'features': featurelist,
+            'numberMatched': matchcount,
+            'numberReturned': len(featurelist)
+        }
+
+        return feature_collection
+
+    def get(self, identifier):
+        """
+        query the provider by id
+
+        :param identifier: feature id
+        :returns: dict of single GeoJSON feature
+        """
+        featurelist, matchcount = self._get_feature_list(
+                                    {'_id': ObjectId(identifier)})
+        return featurelist[0] if featurelist else None
+
+    def create(self, new_feature):
+        """Create a new feature
+        """
+        self.featuredb[self.collection].insert_one(new_feature)
+
+    def update(self, identifier, updated_feature):
+        """Updates an existing feature id with new_feature
+
+        :param identifier: feature id
+        :param new_feature: new GeoJSON feature dictionary
+        """
+        data = {k: v for k, v in updated_feature.items() if k != 'id'}
+        self.featuredb[self.collection].update_one(
+            {'_id': ObjectId(identifier)}, {"$set": data})
+
+    def delete(self, identifier):
+        """Delets an existing feature
+
+        :param identifier: feature id
+        """
+        self.featuredb[self.collection].delete_one(
+            {'_id': ObjectId(identifier)})
@@ -1,3 +1,4 @@
 elasticsearch==7.0.4
 GDAL>=2.2,<3.0
 psycopg2==2.7.6
+pymongo
@@ -0,0 +1,52 @@
+# =================================================================
+#
+# Authors: Timo Tuunanen <timo.tuunanen@rdvelho.com>
+#
+# Copyright (c) 2019 Timo Tuunanen
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+
+import json
+import sys
+from pymongo import MongoClient
+from pymongo import GEOSPHERE
+
+monogourl = 'mongodb://localhost:27017/'
+mongodb = 'testdb'
+mongocollection = 'testplaces'
+
+if len(sys.argv) == 1:
+    print('Usage: {} <path/to/data.geojson>'.format(sys.argv[0]))
+    sys.exit(1)
+
+myclient = MongoClient(monogourl)
+mydb = myclient[mongodb]
+mycol = mydb[mongocollection]
+mycol.drop()
+
+with open(sys.argv[1]) as fh:
+    d = json.load(fh)
+
+mycol.create_index([("geometry", GEOSPHERE)])
+mycol.insert_many(d['features'])
@@ -0,0 +1,219 @@
+# =================================================================
+#
+# Authors: Timo Tuunanen <timo.tuunanen@rdvelho.com>
+#
+# Copyright (c) 2019 Timo Tuunanen
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+
+import pytest
+
+from pygeoapi.provider.mongo import MongoProvider
+
+monogourl = 'mongodb://localhost:27017/testdb'
+mongocollection = 'testplaces'
+
+
+@pytest.fixture()
+def config():
+    return {
+        'name': 'MongoDB',
+        'data': monogourl,
+        'collection': mongocollection
+    }
+
+
+def delete_by_name(provided, name):
+    # delete all existing
+    results = provided.query(properties=[('name', name)])
+    for res in results['features']:
+        provided.delete(res['id'])
+
+
+def init(provider):
+    # delete all existing Unit Test and Null Islands
+    delete_by_name(provider, 'Unit Test Island')
+    delete_by_name(provider, 'Null Island')
+
+
+def test_query(config):
+    p = MongoProvider(config)
+    init(p)
+    results = p.query()
+    assert len(results['features']) == 10
+    assert results['numberMatched'] == 243
+    assert results['numberReturned'] == 10
+    assert results['features'][0]['properties']['nameascii'] == 'Vatican City'
+
+    results = p.query(properties=[('nameascii', 'Vatican City')])
+    assert len(results['features']) == 1
+    assert results['numberMatched'] == 1
+    assert results['numberReturned'] == 1
+    assert results['features'][0]['properties']['nameascii'] == 'Vatican City'
+
+    results = p.query(limit=1)
+    assert len(results['features']) == 1
+    assert results['features'][0]['properties']['nameascii'] == 'Vatican City'
+
+    results = p.query(startindex=2, limit=1)
+    assert len(results['features']) == 1
+    assert results['features'][0]['properties']['nameascii'] == 'Vaduz'
+
+    results = p.query(sortby=[{'property': 'nameascii', 'order': 'A'}])
+    assert results['features'][0]['properties']['nameascii'] == 'Abidjan'
+
+    results = p.query(sortby=[{'property': 'nameascii', 'order': 'D'}])
+    assert results['features'][0]['properties']['nameascii'] == 'Zagreb'
+
+    results = p.query(sortby=[{'property': 'scalerank', 'order': 'A'}])
+    assert results['features'][0]['properties']['scalerank'] == 0
+
+    results = p.query(sortby=[{'property': 'scalerank', 'order': 'D'}])
+    assert results['features'][0]['properties']['scalerank'] == 8
+
+    print(results['features'][0])
+    print(len(results['features'][0]['properties']))
+    assert len(results['features'][0]['properties']) == 37
+
+
+def test_get(config):
+    p = MongoProvider(config)
+    init(p)
+    results = p.get('123456789012345678901234')
+    assert results is None
+
+    res = p.query(properties=[['nameascii', 'Reykjavik']])
+    result = p.get(res['features'][0]['id'])
+    assert isinstance(result, dict)
+    assert 'Reykjavik' in result['properties']['ls_name']
+
+
+def test_get_fields(config):
+    p = MongoProvider(config)
+    init(p)
+    results = p.get_fields()
+    assert len(results) == 37
+
+
+def test_create_and_delete(config):
+    p = MongoProvider(config)
+    init(p)
+
+    new_feature = {
+        'type': 'Feature',
+        'geometry': {
+            'type': 'Point',
+            'coordinates': [0.0, 0.0]},
+        'properties': {
+            'name': 'Null Island'}}
+
+    p.create(new_feature)
+    results = p.query()
+    assert results['numberMatched'] == 244
+
+    results = p.query(properties=[('name', 'Null Island')])
+    assert len(results['features']) == 1
+    assert 'Null Island' in results['features'][0]['properties']['name']
+
+    p.delete(results['features'][0]['id'])
+
+    results = p.query(properties=[('name', 'Null Island')])
+    assert len(results['features']) == 0
+
+    results = p.query()
+    assert results['numberMatched'] == 243
+
+
+def test_update(config):
+    p = MongoProvider(config)
+    init(p)
+
+    new_feature = {
+        'type': 'Feature',
+        'geometry': {
+            'type': 'Point',
+            'coordinates': [0.0, 0.0]},
+        'properties': {
+            'name': 'Unit Test Island'}}
+
+    p.create(new_feature)
+
+    res = p.query(properties=[('name', 'Unit Test Island')])
+    assert len(res['features']) == 1
+
+    updated_feature = {
+        'type': 'Feature',
+        'geometry': {
+            'type': 'Point',
+            'coordinates': [0.0, 0.0]},
+        'properties': {
+            'name': 'Null Island'
+        }
+    }
+
+    p.update(res['features'][0]['id'], updated_feature)
+
+    # Should be changed
+    results = p.get(res['features'][0]['id'])
+    assert 'Null Island' in results['properties']['name']
+    delete_by_name(p, 'Null Island')
+
+
+def test_update_safe_id(config):
+    p = MongoProvider(config)
+    init(p)
+
+    new_feature = {
+        'type': 'Feature',
+        'geometry': {
+            'type': 'Point',
+            'coordinates': [0.0, 0.0]},
+        'properties': {
+            'name': 'Unit Test Island'}}
+
+    p.create(new_feature)
+
+    updated_feature = {
+        'type': 'Feature',
+        'geometry': {
+            'type': 'Point',
+            'coordinates': [0.0, 0.0]
+        },
+        'properties': {
+            'name': 'Null Island',
+        },
+        'id': '123456789012345678901234'
+    }
+
+    res = p.query(properties=[('name', 'Unit Test Island')])
+    assert len(res['features']) == 1
+    p.update(res['features'][0]['id'], updated_feature)
+
+    # Don't let the id change, should not exist
+    assert p.get('123456789012345678901234') is None
+
+    # Should still be at the old id
+    results = p.get(res['features'][0]['id'])
+    assert 'Null Island' in results['properties']['name']
+    delete_by_name(p, 'Null Island')