huanld/pygeoapi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Elasticsearch] update to support for ES 7 (#336)

Browse Source
This commit is contained in:
Tom Kralidis
2020-01-12 21:07:21 -05:00
committed by GitHub
parent 29d119c419
commit 56500e6155
10 changed files with 46 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+9 -1
View File
@@ -14,7 +14,6 @@ python:
- "3.7"
services:
- elasticsearch
- postgresql
- mongodb
@@ -30,6 +29,15 @@ before_install:
- sudo apt-get install -y libgdal-dev gdal-bin
- sudo apt-get install -y mongodb
# install Elasticsearch 7
- curl -s -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.5.1-amd64.deb
- sudo dpkg -i --force-confnew elasticsearch-7.5.1-amd64.deb
- sudo sed -i.old 's/-Xms1g/-Xms128m/' /etc/elasticsearch/jvm.options
- sudo sed -i.old 's/-Xmx1g/-Xmx128m/' /etc/elasticsearch/jvm.options
- echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlog4j.skipJansi=true\n-server\n' | sudo tee -a /etc/elasticsearch/jvm.options
- sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch
- sudo systemctl start elasticsearch
install:
# follow GDAL installed version for Python bindings
- pip3 install GDAL==`gdalinfo --version | cut -d' ' -f2 | cut -d',' -f1`
aws-lambda/pygeoapi-serverless-config.yml
+2 -1
View File
@@ -3,6 +3,7 @@
# Authors: David Bitner <bitner@dbspatial.com>>
#
# Copyright (c) 2019 David Bitner
# Copyright (c) 2020 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -138,7 +139,7 @@ datasets:
crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
provider:
name: Elasticsearch
data: http://localhost:9200/ne_110m_populated_places_simple/FeatureCollection
data: http://localhost:9200/ne_110m_populated_places_simple
id_field: geonameid
lakes:
docker/default.config.yml
+3 -1
View File
@@ -1,8 +1,10 @@
# =================================================================
#
# Authors: Just van den Broecke <justb4@gmail.com>
# Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2019 Just van den Broecke
# Copyright (c) 2020 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -141,7 +143,7 @@ datasets:
# crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
# provider:
# name: Elasticsearch
# data: http://localhost:9200/ne_110m_populated_places_simple/FeatureCollection
# data: http://localhost:9200/ne_110m_populated_places_simple
# id_field: geonameid
#
lakes:
docker/examples/elastic/ES/Dockerfile
+3 -1
View File
@@ -2,9 +2,11 @@
#
# Authors: Just van den Broecke <justb4@gmail.com>>
# Jorge Samuel Mendes de Jesus <jorge.dejesus@geocat.net>
# Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2019 Just van den Broecke
# Copyright (c) 2019 Jorge Samuel Mendes de Jesus
# Copyright (c) 2020 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -29,7 +31,7 @@
#
# =================================================================
FROM docker.elastic.co/elasticsearch/elasticsearch:5.6.8
FROM docker.elastic.co/elasticsearch/elasticsearch:7.5.1
LABEL maintainer="jorge.dejesus@geocat.net justb4@gmail.com"
ARG DATA_FOLDER=/usr/share/elasticsearch/data
docker/examples/elastic/pygeoapi/docker.config.yml
+3 -1
View File
@@ -2,9 +2,11 @@
#
# Authors: Just van den Broecke <justb4@gmail.com>>
# Jorge Samuel Mendes de Jesus <jorge.dejesus@geocat.net>
# Tom Kraldis <tomkralidis@gmail.com>
#
# Copyright (c) 2019 Just van den Broecke
# Copyright (c) 2019 Jorge Samuel Mendes de Jesus
# Copyright (c) 2020 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -141,7 +143,7 @@ datasets:
provider:
name: Elasticsearch
#Note elastic_search is the docker container of ES the name is defined in the docker-compose.yml
data: http://elastic_search:9200/ne_110m_populated_places_simple/FeatureCollection
data: http://elastic_search:9200/ne_110m_populated_places_simple
id_field: geonameid
lakes:
pygeoapi/provider/elasticsearch_.py
+5 -8
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
# Copyright (c) 2020 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -57,11 +57,9 @@ class ElasticsearchProvider(BaseProvider):
LOGGER.debug('Setting Elasticsearch properties')
self.es_host = url_tokens[2]
self.index_name = url_tokens[-2]
self.type_name = url_tokens[-1]
self.index_name = url_tokens[-1]
LOGGER.debug('host: {}'.format(self.es_host))
LOGGER.debug('index: {}'.format(self.index_name))
LOGGER.debug('type: {}'.format(self.type_name))
LOGGER.debug('Connecting to Elasticsearch')
self.es = Elasticsearch(self.es_host)
@@ -87,7 +85,7 @@ class ElasticsearchProvider(BaseProvider):
fields_ = {}
ic = IndicesClient(self.es)
ii = ic.get(self.index_name)
p = ii[self.index_name]['mappings'][self.type_name]['properties']['properties'] # noqa
p = ii[self.index_name]['mappings']['properties']['properties'] # noqa
for k, v in p['properties'].items():
if 'type' in v:
@@ -254,7 +252,7 @@ class ElasticsearchProvider(BaseProvider):
LOGGER.error(err)
raise ProviderQueryError()
feature_collection['numberMatched'] = results['hits']['total']
feature_collection['numberMatched'] = results['hits']['total']['value']
if resulttype == 'hits':
return feature_collection
@@ -298,8 +296,7 @@ class ElasticsearchProvider(BaseProvider):
try:
LOGGER.debug('Fetching identifier {}'.format(identifier))
result = self.es.get(self.index_name, doc_type=self.type_name,
id=identifier)
result = self.es.get(self.index_name, id=identifier)
LOGGER.debug('Serializing feature')
id_ = result['_source']['properties'][self.id_field]
result['_source']['id'] = id_
requirements-provider.txt
+1 -1
View File
@@ -1,4 +1,4 @@
elasticsearch==7.0.4
elasticsearch==7.1.0
GDAL>=2.2,<3.0
psycopg2==2.7.6
pymongo
tests/cite/ogcapi-features/cite.config.yml
+2 -2
View File
@@ -15,7 +15,7 @@ server:
attribution: '<a href="https://wikimediafoundation.org/wiki/Maps_Terms_of_Use">Wikimedia maps</a> | Map data &copy; <a href="https://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
logging:
level: ERROR
level: DEBUG
#logfile: /tmp/pygeoapi.log
metadata:
@@ -94,6 +94,6 @@ datasets:
provider:
name: Elasticsearch
data: http://localhost:9200/canada-hydat-daily-mean-02hc003/FeatureCollection
data: http://localhost:9200/canada-hydat-daily-mean-02hc003
id_field: IDENTIFIER
time_field: DATE
tests/load_es_data.py
+15 -16
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
# Copyright (c) 2020 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -34,8 +34,6 @@ import sys
from elasticsearch import Elasticsearch
es = Elasticsearch()
type_name = 'FeatureCollection'
if len(sys.argv) < 3:
print('Usage: {} <path/to/data.geojson> <id-field>'.format(sys.argv[0]))
sys.exit(1)
@@ -48,20 +46,22 @@ if es.indices.exists(index_name):
# index settings
settings = {
'settings': {
'number_of_shards': 1,
'number_of_replicas': 0
},
'mappings': {
'FeatureCollection': {
'properties': {
'geometry': {
'type': 'geo_shape'
},
'properties': {
'geometry': {
'type': 'geo_shape'
},
'properties': {
'properties': {
'nameascii': {
'type': 'text',
'fields': {
'raw': {
'type': 'keyword'
}
'nameascii': {
'type': 'text',
'fields': {
'raw': {
'type': 'keyword'
}
}
}
@@ -82,5 +82,4 @@ for f in d['features']:
f['properties'][id_field] = int(f['properties'][id_field])
except ValueError:
f['properties'][id_field] = f['properties'][id_field]
res = es.index(index=index_name, doc_type=type_name,
id=f['properties'][id_field], body=f)
res = es.index(index=index_name, id=f['properties'][id_field], body=f)
tests/test_elasticsearch__provider.py
+3 -3
View File
@@ -2,7 +2,7 @@
#
# Authors: Tom Kralidis <tomkralidis@gmail.com>
#
# Copyright (c) 2018 Tom Kralidis
# Copyright (c) 2020 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
@@ -36,7 +36,7 @@ from pygeoapi.provider.elasticsearch_ import ElasticsearchProvider
def config():
return {
'name': 'Elasticsearch',
'data': 'http://localhost:9200/ne_110m_populated_places_simple/FeatureCollection', # noqa
'data': 'http://localhost:9200/ne_110m_populated_places_simple', # noqa
'id_field': 'geonameid'
}
@@ -61,7 +61,7 @@ def test_query(config):
results = p.query(startindex=2, limit=1)
assert len(results['features']) == 1
assert results['features'][0]['id'] == 1559804
assert results['features'][0]['id'] == 3042030
results = p.query(sortby=[{'property': 'nameascii', 'order': 'A'}])
assert results['features'][0]['properties']['nameascii'] == 'Abidjan'