diff --git a/.travis.yml b/.travis.yml index 2b8e2ec..be95047 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,7 @@ python: services: - elasticsearch - postgresql + - mongodb addons: postgresql: 9.6 @@ -27,6 +28,7 @@ before_install: - sudo apt-get install -y libsqlite3-mod-spatialite devscripts fakeroot debhelper - sudo apt-get install -y postgresql-9.6-postgis-2.4 - sudo apt-get install -y libgdal-dev gdal-bin + - sudo apt-get install -y mongodb install: # follow GDAL installed version for Python bindings @@ -39,6 +41,7 @@ install: before_script: - sleep 20 - python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid + - python3 tests/load_mongo_data.py tests/data/ne_110m_populated_places_simple.geojson - pygeoapi generate-openapi-document -c pygeoapi-config.yml > pygeoapi-openapi.yml - psql -U postgres -c 'create database test' - psql -U postgres -d test -c 'create extension postgis' diff --git a/pygeoapi/plugin.py b/pygeoapi/plugin.py index 110a856..2cf90ab 100644 --- a/pygeoapi/plugin.py +++ b/pygeoapi/plugin.py @@ -43,7 +43,8 @@ PLUGINS = { 'GeoPackage': 'pygeoapi.provider.geopackage.GeoPackageProvider', 'OGR': 'pygeoapi.provider.ogr.OGRProvider', 'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider', - 'SQLiteGPKG': 'pygeoapi.provider.sqlite.SQLiteGPKGProvider' + 'SQLiteGPKG': 'pygeoapi.provider.sqlite.SQLiteGPKGProvider', + 'MongoDB': 'pygeoapi.provider.mongo.MongoProvider' }, 'formatter': { 'CSV': 'pygeoapi.formatter.csv_.CSVFormatter' diff --git a/pygeoapi/provider/mongo.py b/pygeoapi/provider/mongo.py new file mode 100644 index 0000000..ae1bed3 --- /dev/null +++ b/pygeoapi/provider/mongo.py @@ -0,0 +1,175 @@ +# ================================================================= +# +# Authors: Timo Tuunanen +# +# Copyright (c) 2019 Timo Tuunanen +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging + +from bson import Code +from pymongo import MongoClient +from pymongo import GEOSPHERE +from pymongo import ASCENDING, DESCENDING +from pymongo.collection import ObjectId +from pygeoapi.provider.base import BaseProvider + +LOGGER = logging.getLogger(__name__) + + +class MongoProvider(BaseProvider): + """Generic provider for Mongodb. + """ + + def __init__(self, provider_def): + """ + MongoProvider Class constructor + + :param provider_def: provider definitions from yml pygeoapi-config. + data,id_field, name set in parent class + + :returns: pygeoapi.providers.mongo.MongoProvider + """ + # this is dummy value never used in case of Mongo. + # Mongo id field is _id + provider_def.setdefault('id_field', '_id') + + BaseProvider.__init__(self, provider_def) + + LOGGER.info('Mongo source config: {}'.format(self.data)) + + dbclient = MongoClient(self.data) + self.featuredb = dbclient.get_default_database() + self.collection = provider_def['collection'] + self.featuredb[self.collection].create_index([("geometry", GEOSPHERE)]) + + def get_fields(self): + """ + Get provider field information (names, types) + + :returns: dict of fields + """ + map = Code( + "function() { for (var key in this.properties) " + "{ emit(key, null); } }") + reduce = Code("function(key, stuff) { return null; }") + result = self.featuredb[self.collection].map_reduce( + map, reduce, "myresults") + return result.distinct('_id') + + def _get_feature_list(self, filterObj, sortList=[], skip=0, maxitems=1): + featurecursor = self.featuredb[self.collection].find(filterObj) + + if sortList: + featurecursor = featurecursor.sort(sortList) + + matchCount = self.featuredb[self.collection].count_documents(filterObj) + featurecursor.skip(skip) + featurecursor.limit(maxitems) + featurelist = list(featurecursor) + for item in featurelist: + item['id'] = str(item.pop('_id')) + + return featurelist, matchCount + + def query(self, startindex=0, limit=10, resulttype='results', + bbox=[], datetime=None, properties=[], sortby=[]): + """ + query the provider + + :returns: dict of 0..n GeoJSON features + """ + and_filter = [] + + if len(bbox) == 4: + x, y, w, h = map(float, bbox) + and_filter.append( + {'geometry': {'$geoWithin': {'$box': [[x, y], [w, h]]}}}) + + # This parameter is not working yet! + # gte is not sufficient to check date range + if datetime is not None: + assert isinstance(datetime.datetime, datetime) + and_filter.append({'properties.datetime': {'$gte': datetime}}) + + for prop in properties: + and_filter.append({"properties."+prop[0]: {'$eq': prop[1]}}) + + filterobj = {'$and': and_filter} if and_filter else {} + + sort_list = [("properties." + sort['property'], + ASCENDING if (sort['order'] == 'A') else DESCENDING) + for sort in sortby] + + featurelist, matchcount = self._get_feature_list(filterobj, + sortList=sort_list, + skip=startindex, + maxitems=limit) + + if resulttype == 'hits': + featurelist = [] + + feature_collection = { + 'type': 'FeatureCollection', + 'features': featurelist, + 'numberMatched': matchcount, + 'numberReturned': len(featurelist) + } + + return feature_collection + + def get(self, identifier): + """ + query the provider by id + + :param identifier: feature id + :returns: dict of single GeoJSON feature + """ + featurelist, matchcount = self._get_feature_list( + {'_id': ObjectId(identifier)}) + return featurelist[0] if featurelist else None + + def create(self, new_feature): + """Create a new feature + """ + self.featuredb[self.collection].insert_one(new_feature) + + def update(self, identifier, updated_feature): + """Updates an existing feature id with new_feature + + :param identifier: feature id + :param new_feature: new GeoJSON feature dictionary + """ + data = {k: v for k, v in updated_feature.items() if k != 'id'} + self.featuredb[self.collection].update_one( + {'_id': ObjectId(identifier)}, {"$set": data}) + + def delete(self, identifier): + """Delets an existing feature + + :param identifier: feature id + """ + self.featuredb[self.collection].delete_one( + {'_id': ObjectId(identifier)}) diff --git a/requirements-provider.txt b/requirements-provider.txt index b2613c5..51df908 100644 --- a/requirements-provider.txt +++ b/requirements-provider.txt @@ -1,3 +1,4 @@ elasticsearch==7.0.4 GDAL>=2.2,<3.0 psycopg2==2.7.6 +pymongo diff --git a/tests/load_mongo_data.py b/tests/load_mongo_data.py new file mode 100644 index 0000000..b6aa914 --- /dev/null +++ b/tests/load_mongo_data.py @@ -0,0 +1,52 @@ +# ================================================================= +# +# Authors: Timo Tuunanen +# +# Copyright (c) 2019 Timo Tuunanen +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import json +import sys +from pymongo import MongoClient +from pymongo import GEOSPHERE + +monogourl = 'mongodb://localhost:27017/' +mongodb = 'testdb' +mongocollection = 'testplaces' + +if len(sys.argv) == 1: + print('Usage: {} '.format(sys.argv[0])) + sys.exit(1) + +myclient = MongoClient(monogourl) +mydb = myclient[mongodb] +mycol = mydb[mongocollection] +mycol.drop() + +with open(sys.argv[1]) as fh: + d = json.load(fh) + +mycol.create_index([("geometry", GEOSPHERE)]) +mycol.insert_many(d['features']) diff --git a/tests/test_mongo_provider.py b/tests/test_mongo_provider.py new file mode 100644 index 0000000..87abcd5 --- /dev/null +++ b/tests/test_mongo_provider.py @@ -0,0 +1,219 @@ +# ================================================================= +# +# Authors: Timo Tuunanen +# +# Copyright (c) 2019 Timo Tuunanen +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import pytest + +from pygeoapi.provider.mongo import MongoProvider + +monogourl = 'mongodb://localhost:27017/testdb' +mongocollection = 'testplaces' + + +@pytest.fixture() +def config(): + return { + 'name': 'MongoDB', + 'data': monogourl, + 'collection': mongocollection + } + + +def delete_by_name(provided, name): + # delete all existing + results = provided.query(properties=[('name', name)]) + for res in results['features']: + provided.delete(res['id']) + + +def init(provider): + # delete all existing Unit Test and Null Islands + delete_by_name(provider, 'Unit Test Island') + delete_by_name(provider, 'Null Island') + + +def test_query(config): + p = MongoProvider(config) + init(p) + results = p.query() + assert len(results['features']) == 10 + assert results['numberMatched'] == 243 + assert results['numberReturned'] == 10 + assert results['features'][0]['properties']['nameascii'] == 'Vatican City' + + results = p.query(properties=[('nameascii', 'Vatican City')]) + assert len(results['features']) == 1 + assert results['numberMatched'] == 1 + assert results['numberReturned'] == 1 + assert results['features'][0]['properties']['nameascii'] == 'Vatican City' + + results = p.query(limit=1) + assert len(results['features']) == 1 + assert results['features'][0]['properties']['nameascii'] == 'Vatican City' + + results = p.query(startindex=2, limit=1) + assert len(results['features']) == 1 + assert results['features'][0]['properties']['nameascii'] == 'Vaduz' + + results = p.query(sortby=[{'property': 'nameascii', 'order': 'A'}]) + assert results['features'][0]['properties']['nameascii'] == 'Abidjan' + + results = p.query(sortby=[{'property': 'nameascii', 'order': 'D'}]) + assert results['features'][0]['properties']['nameascii'] == 'Zagreb' + + results = p.query(sortby=[{'property': 'scalerank', 'order': 'A'}]) + assert results['features'][0]['properties']['scalerank'] == 0 + + results = p.query(sortby=[{'property': 'scalerank', 'order': 'D'}]) + assert results['features'][0]['properties']['scalerank'] == 8 + + print(results['features'][0]) + print(len(results['features'][0]['properties'])) + assert len(results['features'][0]['properties']) == 37 + + +def test_get(config): + p = MongoProvider(config) + init(p) + results = p.get('123456789012345678901234') + assert results is None + + res = p.query(properties=[['nameascii', 'Reykjavik']]) + result = p.get(res['features'][0]['id']) + assert isinstance(result, dict) + assert 'Reykjavik' in result['properties']['ls_name'] + + +def test_get_fields(config): + p = MongoProvider(config) + init(p) + results = p.get_fields() + assert len(results) == 37 + + +def test_create_and_delete(config): + p = MongoProvider(config) + init(p) + + new_feature = { + 'type': 'Feature', + 'geometry': { + 'type': 'Point', + 'coordinates': [0.0, 0.0]}, + 'properties': { + 'name': 'Null Island'}} + + p.create(new_feature) + results = p.query() + assert results['numberMatched'] == 244 + + results = p.query(properties=[('name', 'Null Island')]) + assert len(results['features']) == 1 + assert 'Null Island' in results['features'][0]['properties']['name'] + + p.delete(results['features'][0]['id']) + + results = p.query(properties=[('name', 'Null Island')]) + assert len(results['features']) == 0 + + results = p.query() + assert results['numberMatched'] == 243 + + +def test_update(config): + p = MongoProvider(config) + init(p) + + new_feature = { + 'type': 'Feature', + 'geometry': { + 'type': 'Point', + 'coordinates': [0.0, 0.0]}, + 'properties': { + 'name': 'Unit Test Island'}} + + p.create(new_feature) + + res = p.query(properties=[('name', 'Unit Test Island')]) + assert len(res['features']) == 1 + + updated_feature = { + 'type': 'Feature', + 'geometry': { + 'type': 'Point', + 'coordinates': [0.0, 0.0]}, + 'properties': { + 'name': 'Null Island' + } + } + + p.update(res['features'][0]['id'], updated_feature) + + # Should be changed + results = p.get(res['features'][0]['id']) + assert 'Null Island' in results['properties']['name'] + delete_by_name(p, 'Null Island') + + +def test_update_safe_id(config): + p = MongoProvider(config) + init(p) + + new_feature = { + 'type': 'Feature', + 'geometry': { + 'type': 'Point', + 'coordinates': [0.0, 0.0]}, + 'properties': { + 'name': 'Unit Test Island'}} + + p.create(new_feature) + + updated_feature = { + 'type': 'Feature', + 'geometry': { + 'type': 'Point', + 'coordinates': [0.0, 0.0] + }, + 'properties': { + 'name': 'Null Island', + }, + 'id': '123456789012345678901234' + } + + res = p.query(properties=[('name', 'Unit Test Island')]) + assert len(res['features']) == 1 + p.update(res['features'][0]['id'], updated_feature) + + # Don't let the id change, should not exist + assert p.get('123456789012345678901234') is None + + # Should still be at the old id + results = p.get(res['features'][0]['id']) + assert 'Null Island' in results['properties']['name'] + delete_by_name(p, 'Null Island')