diff --git a/.travis.yml b/.travis.yml index be95047..f79939e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,6 @@ python: - "3.7" services: - - elasticsearch - postgresql - mongodb @@ -30,6 +29,15 @@ before_install: - sudo apt-get install -y libgdal-dev gdal-bin - sudo apt-get install -y mongodb + # install Elasticsearch 7 + - curl -s -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.5.1-amd64.deb + - sudo dpkg -i --force-confnew elasticsearch-7.5.1-amd64.deb + - sudo sed -i.old 's/-Xms1g/-Xms128m/' /etc/elasticsearch/jvm.options + - sudo sed -i.old 's/-Xmx1g/-Xmx128m/' /etc/elasticsearch/jvm.options + - echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlog4j.skipJansi=true\n-server\n' | sudo tee -a /etc/elasticsearch/jvm.options + - sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch + - sudo systemctl start elasticsearch + install: # follow GDAL installed version for Python bindings - pip3 install GDAL==`gdalinfo --version | cut -d' ' -f2 | cut -d',' -f1` diff --git a/aws-lambda/pygeoapi-serverless-config.yml b/aws-lambda/pygeoapi-serverless-config.yml index 8c18f37..cfe20b0 100644 --- a/aws-lambda/pygeoapi-serverless-config.yml +++ b/aws-lambda/pygeoapi-serverless-config.yml @@ -3,6 +3,7 @@ # Authors: David Bitner > # # Copyright (c) 2019 David Bitner +# Copyright (c) 2020 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -138,7 +139,7 @@ datasets: crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 provider: name: Elasticsearch - data: http://localhost:9200/ne_110m_populated_places_simple/FeatureCollection + data: http://localhost:9200/ne_110m_populated_places_simple id_field: geonameid lakes: diff --git a/docker/default.config.yml b/docker/default.config.yml index c2d2965..d1dd393 100644 --- a/docker/default.config.yml +++ b/docker/default.config.yml @@ -1,8 +1,10 @@ # ================================================================= # # Authors: Just van den Broecke +# Tom Kralidis # # Copyright (c) 2019 Just van den Broecke +# Copyright (c) 2020 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -141,7 +143,7 @@ datasets: # crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 # provider: # name: Elasticsearch -# data: http://localhost:9200/ne_110m_populated_places_simple/FeatureCollection +# data: http://localhost:9200/ne_110m_populated_places_simple # id_field: geonameid # lakes: diff --git a/docker/examples/elastic/ES/Dockerfile b/docker/examples/elastic/ES/Dockerfile index 64fcd9f..a3434da 100644 --- a/docker/examples/elastic/ES/Dockerfile +++ b/docker/examples/elastic/ES/Dockerfile @@ -2,9 +2,11 @@ # # Authors: Just van den Broecke > # Jorge Samuel Mendes de Jesus +# Tom Kralidis # # Copyright (c) 2019 Just van den Broecke # Copyright (c) 2019 Jorge Samuel Mendes de Jesus +# Copyright (c) 2020 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -29,7 +31,7 @@ # # ================================================================= -FROM docker.elastic.co/elasticsearch/elasticsearch:5.6.8 +FROM docker.elastic.co/elasticsearch/elasticsearch:7.5.1 LABEL maintainer="jorge.dejesus@geocat.net justb4@gmail.com" ARG DATA_FOLDER=/usr/share/elasticsearch/data diff --git a/docker/examples/elastic/pygeoapi/docker.config.yml b/docker/examples/elastic/pygeoapi/docker.config.yml index 8eacaad..e699ebc 100644 --- a/docker/examples/elastic/pygeoapi/docker.config.yml +++ b/docker/examples/elastic/pygeoapi/docker.config.yml @@ -2,9 +2,11 @@ # # Authors: Just van den Broecke > # Jorge Samuel Mendes de Jesus +# Tom Kraldis # # Copyright (c) 2019 Just van den Broecke # Copyright (c) 2019 Jorge Samuel Mendes de Jesus +# Copyright (c) 2020 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -141,7 +143,7 @@ datasets: provider: name: Elasticsearch #Note elastic_search is the docker container of ES the name is defined in the docker-compose.yml - data: http://elastic_search:9200/ne_110m_populated_places_simple/FeatureCollection + data: http://elastic_search:9200/ne_110m_populated_places_simple id_field: geonameid lakes: diff --git a/pygeoapi/provider/elasticsearch_.py b/pygeoapi/provider/elasticsearch_.py index 860bf97..fa77117 100644 --- a/pygeoapi/provider/elasticsearch_.py +++ b/pygeoapi/provider/elasticsearch_.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2020 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -57,11 +57,9 @@ class ElasticsearchProvider(BaseProvider): LOGGER.debug('Setting Elasticsearch properties') self.es_host = url_tokens[2] - self.index_name = url_tokens[-2] - self.type_name = url_tokens[-1] + self.index_name = url_tokens[-1] LOGGER.debug('host: {}'.format(self.es_host)) LOGGER.debug('index: {}'.format(self.index_name)) - LOGGER.debug('type: {}'.format(self.type_name)) LOGGER.debug('Connecting to Elasticsearch') self.es = Elasticsearch(self.es_host) @@ -87,7 +85,7 @@ class ElasticsearchProvider(BaseProvider): fields_ = {} ic = IndicesClient(self.es) ii = ic.get(self.index_name) - p = ii[self.index_name]['mappings'][self.type_name]['properties']['properties'] # noqa + p = ii[self.index_name]['mappings']['properties']['properties'] # noqa for k, v in p['properties'].items(): if 'type' in v: @@ -254,7 +252,7 @@ class ElasticsearchProvider(BaseProvider): LOGGER.error(err) raise ProviderQueryError() - feature_collection['numberMatched'] = results['hits']['total'] + feature_collection['numberMatched'] = results['hits']['total']['value'] if resulttype == 'hits': return feature_collection @@ -298,8 +296,7 @@ class ElasticsearchProvider(BaseProvider): try: LOGGER.debug('Fetching identifier {}'.format(identifier)) - result = self.es.get(self.index_name, doc_type=self.type_name, - id=identifier) + result = self.es.get(self.index_name, id=identifier) LOGGER.debug('Serializing feature') id_ = result['_source']['properties'][self.id_field] result['_source']['id'] = id_ diff --git a/requirements-provider.txt b/requirements-provider.txt index 51df908..29c3121 100644 --- a/requirements-provider.txt +++ b/requirements-provider.txt @@ -1,4 +1,4 @@ -elasticsearch==7.0.4 +elasticsearch==7.1.0 GDAL>=2.2,<3.0 psycopg2==2.7.6 pymongo diff --git a/tests/cite/ogcapi-features/cite.config.yml b/tests/cite/ogcapi-features/cite.config.yml index 3269222..2d937cb 100644 --- a/tests/cite/ogcapi-features/cite.config.yml +++ b/tests/cite/ogcapi-features/cite.config.yml @@ -15,7 +15,7 @@ server: attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' logging: - level: ERROR + level: DEBUG #logfile: /tmp/pygeoapi.log metadata: @@ -94,6 +94,6 @@ datasets: provider: name: Elasticsearch - data: http://localhost:9200/canada-hydat-daily-mean-02hc003/FeatureCollection + data: http://localhost:9200/canada-hydat-daily-mean-02hc003 id_field: IDENTIFIER time_field: DATE diff --git a/tests/load_es_data.py b/tests/load_es_data.py index c70556c..bdd6c52 100644 --- a/tests/load_es_data.py +++ b/tests/load_es_data.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2020 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -34,8 +34,6 @@ import sys from elasticsearch import Elasticsearch es = Elasticsearch() -type_name = 'FeatureCollection' - if len(sys.argv) < 3: print('Usage: {} '.format(sys.argv[0])) sys.exit(1) @@ -48,20 +46,22 @@ if es.indices.exists(index_name): # index settings settings = { + 'settings': { + 'number_of_shards': 1, + 'number_of_replicas': 0 + }, 'mappings': { - 'FeatureCollection': { + 'properties': { + 'geometry': { + 'type': 'geo_shape' + }, 'properties': { - 'geometry': { - 'type': 'geo_shape' - }, 'properties': { - 'properties': { - 'nameascii': { - 'type': 'text', - 'fields': { - 'raw': { - 'type': 'keyword' - } + 'nameascii': { + 'type': 'text', + 'fields': { + 'raw': { + 'type': 'keyword' } } } @@ -82,5 +82,4 @@ for f in d['features']: f['properties'][id_field] = int(f['properties'][id_field]) except ValueError: f['properties'][id_field] = f['properties'][id_field] - res = es.index(index=index_name, doc_type=type_name, - id=f['properties'][id_field], body=f) + res = es.index(index=index_name, id=f['properties'][id_field], body=f) diff --git a/tests/test_elasticsearch__provider.py b/tests/test_elasticsearch__provider.py index 8a6c07d..fa1b6b3 100644 --- a/tests/test_elasticsearch__provider.py +++ b/tests/test_elasticsearch__provider.py @@ -2,7 +2,7 @@ # # Authors: Tom Kralidis # -# Copyright (c) 2018 Tom Kralidis +# Copyright (c) 2020 Tom Kralidis # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation @@ -36,7 +36,7 @@ from pygeoapi.provider.elasticsearch_ import ElasticsearchProvider def config(): return { 'name': 'Elasticsearch', - 'data': 'http://localhost:9200/ne_110m_populated_places_simple/FeatureCollection', # noqa + 'data': 'http://localhost:9200/ne_110m_populated_places_simple', # noqa 'id_field': 'geonameid' } @@ -61,7 +61,7 @@ def test_query(config): results = p.query(startindex=2, limit=1) assert len(results['features']) == 1 - assert results['features'][0]['id'] == 1559804 + assert results['features'][0]['id'] == 3042030 results = p.query(sortby=[{'property': 'nameascii', 'order': 'A'}]) assert results['features'][0]['properties']['nameascii'] == 'Abidjan'