From ac73e95398aa3828009af7629c6342faff59f61d Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Mon, 23 Dec 2019 12:33:31 -0500 Subject: [PATCH] abstract ES test loader to accept id field as argument (#324) --- .travis.yml | 2 +- docker/examples/elastic/ES/add_data.sh | 2 +- tests/load_es_data.py | 14 +++++++++----- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index b63b036..2b8e2ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,7 +38,7 @@ install: before_script: - sleep 20 - - python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson + - python3 tests/load_es_data.py tests/data/ne_110m_populated_places_simple.geojson geonameid - pygeoapi generate-openapi-document -c pygeoapi-config.yml > pygeoapi-openapi.yml - psql -U postgres -c 'create database test' - psql -U postgres -d test -c 'create extension postgis' diff --git a/docker/examples/elastic/ES/add_data.sh b/docker/examples/elastic/ES/add_data.sh index 9ab41d9..de46a5d 100755 --- a/docker/examples/elastic/ES/add_data.sh +++ b/docker/examples/elastic/ES/add_data.sh @@ -44,7 +44,7 @@ until $(curl -sSf -XGET --insecure 'http://localhost:9200/_cluster/health?wait_f done echo "Elasticsearch seems to be working - Adding ne_110m_populated_places_simple.geojson to ES" -python /load_es_data.py /usr/share/elasticsearch/data/ne_110m_populated_places_simple.geojson +python /load_es_data.py /usr/share/elasticsearch/data/ne_110m_populated_places_simple.geojson geonameid echo "Seems that data was loaded" diff --git a/tests/load_es_data.py b/tests/load_es_data.py index 31313ba..d4240fd 100644 --- a/tests/load_es_data.py +++ b/tests/load_es_data.py @@ -36,11 +36,12 @@ es = Elasticsearch() type_name = 'FeatureCollection' -if len(sys.argv) == 1: - print('Usage: {} '.format(sys.argv[0])) +if len(sys.argv) == 2: + print('Usage: {} '.format(sys.argv[0])) sys.exit(1) -index_name = os.path.splitext(os.path.basename(sys.argv[1]))[0] +index_name = os.path.splitext(os.path.basename(sys.argv[1]))[0].lower() +id_field = sys.argv[2] if es.indices.exists(index_name): es.indices.delete(index_name) @@ -77,6 +78,9 @@ with open(sys.argv[1]) as fh: d = json.load(fh) for f in d['features']: - f['properties']['geonameid'] = int(f['properties']['geonameid']) + try: + f['properties'][id_field] = int(f['properties'][id_field]) + except ValueError: + f['properties'][id_field] = f['properties'][id_field] res = es.index(index=index_name, doc_type=type_name, - id=f['properties']['geonameid'], body=f) + id=f['properties'][id_field], body=f)