diff --git a/.gitignore b/.gitignore index 5464689..29c4aff 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,8 @@ pygeoapi/db.sqlite3 # Pycharm project files .idea + +# ES data folder +.pygeoapi/docker/examples/elastic/ES/data +.pygeoapi/docker/examples/mvt-elastic/ES/data +.pygeoapi/docker/examples/mvt-tippecanoe/ES/data \ No newline at end of file diff --git a/docker/examples/README.md b/docker/examples/README.md index d928371..2ce4962 100644 --- a/docker/examples/README.md +++ b/docker/examples/README.md @@ -4,10 +4,30 @@ This folder contains the sub-folders: - simple - elastic -- sensorthings +- esri - mongo +- mvt-elastic +- mvt-tippecanoe +- sensorthings +- skin +- socrata The [simple](simple) example will run pygeoapi with Docker with your local config. + The [elastic](elastic) example demonstrates a docker compose configuration to run pygeoapi with local Elasticsearch backend. + +The [esri](esri) example demonstrates a docker compose configuration to run pygeoapi with ESRI Map and Feature Services backend. + The [mongo](mongo) example demonstrates a docker compose configuration to run pygeoapi with local MongoDB backend. + +The [mvt-elastic](mvt-elastic) example demonstrates a docker compose configuration to run pygeoapi with local Elasticsearch MVT backend. + +The [mvt-tippecanoe](mvt-tippecanoe) example demonstrates a docker compose configuration to run pygeoapi with tiles on disk, pre-generated using the Tippecanoe backend. + The [sensorthings](sensorthings) example demonstrates various pygeoapi implementations of SensorThings API endpoints. + +The [skin](skin) example contains a Docker build script necessary to setup a minimal +pygeoapi server that uses a customised dashboard skin. + +The [socrata](socrata) example contains configuration necessary to setup a +pygeoapi server using a remote Socrata Open Data API (SODA) endpoint. \ No newline at end of file diff --git a/docker/examples/elastic/pygeoapi/docker.config.yml b/docker/examples/elastic/pygeoapi/docker.config.yml index b909fb3..428c77e 100644 --- a/docker/examples/elastic/pygeoapi/docker.config.yml +++ b/docker/examples/elastic/pygeoapi/docker.config.yml @@ -150,21 +150,6 @@ resources: #Note elastic_search is the docker container of ES the name is defined in the docker-compose.yml data: http://elastic_search:9200/ne_110m_populated_places_simple id_field: geonameid - - type: tile - name: MVT - data: http://elastic_search:9200/ne_110m_populated_places_simple/_mvt/geometry/{z}/{x}/{y}?grid_precision=0 - # index must have a geo_point - options: - metadata_format: none # default | tilejson - zoom: - min: 0 - max: 16 - schemes: - - WorldCRS84Quad - format: - name: pbf - mimetype: application/vnd.mapbox-vector-tile - lakes: type: collection diff --git a/docker/examples/mvt-elastic/ES/Dockerfile b/docker/examples/mvt-elastic/ES/Dockerfile new file mode 100644 index 0000000..a4ff36b --- /dev/null +++ b/docker/examples/mvt-elastic/ES/Dockerfile @@ -0,0 +1,73 @@ +# ================================================================= +# +# Authors: Just van den Broecke > +# Jorge Samuel Mendes de Jesus +# Tom Kralidis +# +# Copyright (c) 2019 Just van den Broecke +# Copyright (c) 2019 Jorge Samuel Mendes de Jesus +# Copyright (c) 2020 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +FROM docker.elastic.co/elasticsearch/elasticsearch:8.4.0 + +LABEL maintainer="jorge.dejesus@geocat.net justb4@gmail.com" +ARG DATA_FOLDER=/usr/share/elasticsearch/data + +USER root + +COPY docker-entrypoint.sh /docker-entrypoint.sh +COPY add_data.sh /add_data.sh + +RUN apt update && apt install -y wget + +RUN wget https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -O bin/wait-for-it.sh +RUN chmod +x bin/wait-for-it.sh +RUN wget https://raw.githubusercontent.com/geopython/pygeoapi/master/tests/data/ne_110m_populated_places_simple.geojson -O ${DATA_FOLDER}/ne_110m_populated_places_simple.geojson +RUN wget https://raw.githubusercontent.com/geopython/pygeoapi/master/tests/load_es_data.py -O /load_es_data.py + +RUN echo "xpack.security.enabled: false" >> config/elasticsearch.yml +RUN echo "http.host: 0.0.0.0" >> config/elasticsearch.yml +RUN echo "discovery.type: single-node" >> config/elasticsearch.yml + +RUN apt install -y python3 python3-pip python3-setuptools python-typing \ + && pip3 install --upgrade pip elasticsearch==8.4.0 elasticsearch-dsl \ + && apt clean packages + +USER elasticsearch + +CMD ["/usr/share/elasticsearch/bin/elasticsearch"] + +ENTRYPOINT ["/docker-entrypoint.sh"] + +# we need to run this on host +#sudo sysctl -w vm.max_map_count=262144 +#check indices +#http://localhost:9200/_cat/indices?v +#check spatial data +#http://localhost:9200/ne_110m_populated_places_simple/ +#This docker compose was inspired on: +#https://discuss.elastic.co/t/best-practice-for-creating-an-index-when-an-es-docker-container-starts/126651 +#docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" es:latest diff --git a/docker/examples/mvt-elastic/ES/add_data.sh b/docker/examples/mvt-elastic/ES/add_data.sh new file mode 100755 index 0000000..568dabb --- /dev/null +++ b/docker/examples/mvt-elastic/ES/add_data.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# ================================================================= +# +# Authors: Just van den Broecke > +# Jorge Samuel Mendes de Jesus +# +# Copyright (c) 2019 Just van den Broecke +# Copyright (c) 2019 Jorge Samuel Mendes de Jesus +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + + +echo "Starting script to add geojson" +# move to the directory of this setup script +cd "$(dirname "$0")" + +# for some reason even when port 9200 is open Elasticsearch is unable to be accessed as authentication fails +# a few seconds later it works +# incresing to 50s for wait in a slow system /_cluster/health?wait_for_status=yellow&timeout=50s +until $(curl -sSf -XGET --insecure 'http://localhost:9200/_cluster/health?wait_for_status=yellow' > /dev/null); do + printf 'No status yellow from ES, trying again in 10 seconds \n' + sleep 10 +done +echo "Elasticsearch seems to be working - Adding ne_110m_populated_places_simple.geojson to ES" + +python3 /load_es_data.py /usr/share/elasticsearch/data/ne_110m_populated_places_simple.geojson geonameid + +echo "Seems that data was loaded" + +# create a new index with the settings in es_index_config.json +#curl -v --insecure --user elastic:changeme -XPUT '0.0.0.0:9200/test?pretty' -H 'Content-Type: application/json' -d @es_index_config.json diff --git a/docker/examples/mvt-elastic/ES/docker-entrypoint.sh b/docker/examples/mvt-elastic/ES/docker-entrypoint.sh new file mode 100755 index 0000000..8f586a6 --- /dev/null +++ b/docker/examples/mvt-elastic/ES/docker-entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# ================================================================= +# +# Authors: Just van den Broecke > +# Jorge Samuel Mendes de Jesus +# +# Copyright (c) 2019 Just van den Broecke +# Copyright (c) 2019 Jorge Samuel Mendes de Jesus +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +# wait for Elasticsearch to start, then run the setup script to +# create and configure the index. +exec /usr/share/elasticsearch/bin/wait-for-it.sh localhost:9200 -- /add_data.sh & +exec $@ diff --git a/docker/examples/mvt-elastic/README.md b/docker/examples/mvt-elastic/README.md new file mode 100644 index 0000000..829bbc2 --- /dev/null +++ b/docker/examples/mvt-elastic/README.md @@ -0,0 +1,38 @@ +# pygeoapi with Elastic MVT (ES) + +These folders contain a Docker Compose configuration necessary to setup a minimal +`pygeoapi` server that uses a local ES backend service to publish vector data as both, OGC API - Features and OGC API - Tiles. + +More information about this provider, including the features it supports, can be found on the [pygeoapi documentation](https://docs.pygeoapi.io/en/latest/data-publishing/ogcapi-tiles.html#providers#mvt-elastic). + +This config is only for local development and testing. + +## Elasticsearch + +- official Elasticsearch: **8.4.0** on **Ubuntu 20.04.4 LTS (Focal Fossa)** +- ports **9300** and **9200** + +ES requires the host system to have its virtual memory +parameter (**max_map_count**) [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html) +set as follows: + +``` +sudo sysctl -w vm.max_map_count=262144 +``` + +If the docker composition fails with the following error: +``` +docker_elastic_search_1 exited with code 78 +``` + +it is very likely that you forgot to setup the `sysctl`. + +## Building and Running + +To build and run the [Docker compose file](docker-compose.yml) in localhost: + +``` +sudo sysctl -w vm.max_map_count=262144 +docker-compose build +docker-compose up +``` \ No newline at end of file diff --git a/docker/examples/mvt-elastic/docker-compose.yml b/docker/examples/mvt-elastic/docker-compose.yml new file mode 100644 index 0000000..d1c714a --- /dev/null +++ b/docker/examples/mvt-elastic/docker-compose.yml @@ -0,0 +1,73 @@ +# ================================================================= +# +# Authors: Just van den Broecke > +# Jorge Samuel Mendes de Jesus +# +# Copyright (c) 2019 Just van den Broecke +# Copyright (c) 2019 Jorge Samuel Mendes de Jesus +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +version: '3.3' + +services: + pygeoapi: + image: geopython/pygeoapi:latest + + container_name: pygeoapi_es + + entrypoint: + - /es-entrypoint.sh + + ports: + - 5000:80 + + volumes: + - ./pygeoapi/docker.config.yml:/pygeoapi/local.config.yml + - ./pygeoapi/es-entrypoint.sh:/es-entrypoint.sh + - ./pygeoapi/wait-for-elasticsearch.sh:/wait-for-elasticsearch.sh + + links: + - elastic_search + + depends_on: + - elastic_search + + elastic_search: + build: ./ES + + container_name: elastic +# Elastic ports may be opened for debugging but should remain closed in +# production workloads. + # ports: + # - 9300:9300 + # - 9200:9200 + volumes: + - elastic_search_data:/usr/share/elasticsearch/data + +volumes: + elastic_search_data: {} + +#NOTE: Host requires changes in configuration to run ES +#sudo sysctl -w vm.max_map_count=262144 diff --git a/docker/examples/mvt-elastic/pygeoapi/docker.config.yml b/docker/examples/mvt-elastic/pygeoapi/docker.config.yml new file mode 100644 index 0000000..6ccabb3 --- /dev/null +++ b/docker/examples/mvt-elastic/pygeoapi/docker.config.yml @@ -0,0 +1,128 @@ +# ================================================================= +# +# Authors: Joana Simoes +# +# Copyright (c) 2023 Joana Simoes +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +server: + bind: + host: 0.0.0.0 + port: 5000 + url: http://localhost:5000/ + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + gzip: false + language: en-US + cors: true + pretty_print: true + limit: 10 + # templates: /path/to/templates + map: + url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png + attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + +logging: + level: ERROR + #logfile: /tmp/pygeoapi.log + +metadata: + identification: + title: pygeoapi default instance + description: pygeoapi provides an API to geospatial data + keywords: + - geospatial + - data + - api + keywords_type: theme + terms_of_service: https://creativecommons.org/licenses/by/4.0/ + url: http://example.org + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: Organization Name + url: https://pygeoapi.io + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Hours of Service + instructions: During hours of service. Off on weekends. + role: pointOfContact + +resources: + + ne_110m_populated_places_simple: + type: collection + title: Populated Places + description: Point symbols with name attributes. Includes all admin-0 capitals and some other major cities. We favor regional significance over population census in determining our selection of places. Use the scale rankings to filter the number of towns that appear on your map. + keywords: + - populated places + - cities + - towns + links: + - type: text/html + rel: canonical + title: information + href: http://www.naturalearthdata.com/downloads/110m-cultural-vectors/110m-populated-places/ + hreflang: en-US + - type: application/gzip + rel: canonical + title: download + href: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/ne_110m_populated_places_simple.zip + hreflang: en-US + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: Elasticsearch + #Note elastic_search is the docker container of ES the name is defined in the docker-compose.yml + data: http://elastic_search:9200/ne_110m_populated_places_simple + id_field: geonameid + - type: tile + name: MVT-elastic + data: http://elastic_search:9200/ne_110m_populated_places_simple/_mvt/geometry/{z}/{x}/{y}?grid_precision=0 + # index must have a geo_point + options: + metadata_format: default # default | tilejson + zoom: + min: 0 + max: 16 + schemes: + - WorldCRS84Quad + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile diff --git a/docker/examples/mvt-elastic/pygeoapi/es-entrypoint.sh b/docker/examples/mvt-elastic/pygeoapi/es-entrypoint.sh new file mode 100755 index 0000000..7e8bea1 --- /dev/null +++ b/docker/examples/mvt-elastic/pygeoapi/es-entrypoint.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# ================================================================= +# +# Authors: Just van den Broecke > +# Jorge Samuel Mendes de Jesus +# Tom Kralidis +# +# Copyright (c) 2019 Just van den Broecke +# Copyright (c) 2019 Jorge Samuel Mendes de Jesus +# Copyright (c) 2023 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +set +e + +echo "Install Curl" + +apt-get update -y && +apt-get install curl -y && + +echo "Waiting for Elasticsearch container..." + +# First wait for ES to be up and then execute the original pygeoapi entrypoint. +/wait-for-elasticsearch.sh http://elastic_search:9200 /entrypoint.sh || echo "ES failed: $?, exit" && exit 1 diff --git a/docker/examples/mvt-elastic/pygeoapi/wait-for-elasticsearch.sh b/docker/examples/mvt-elastic/pygeoapi/wait-for-elasticsearch.sh new file mode 100755 index 0000000..06bfce8 --- /dev/null +++ b/docker/examples/mvt-elastic/pygeoapi/wait-for-elasticsearch.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# ================================================================= +# +# Authors: Just van den Broecke > +# Jorge Samuel Mendes de Jesus +# +# Copyright (c) 2019 Just van den Broecke +# Copyright (c) 2019 Jorge Samuel Mendes de Jesus +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +set -e + +host="$1" +shift +cmd="$@" + + +until $(curl --output /dev/null --silent --head --fail "$host") ; do + echo 'Checking if Elasticsearch server is up' + sleep 5 + counter=$((counter+1)) +done + +# First wait for ES to start... +response=$(curl $host) + +until [ "$response" = "200" ] ; do + response=$(curl --write-out %{http_code} --silent --output /dev/null "$host") + >&2 echo "Elasticsearch is up but unavailable - No Reponse - sleeping" + sleep 10 + +done + + +# next wait for ES status to turn to green or yellow +health="$(curl -fsSL "$host/_cat/health?h=status")" +health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ") + +until [ "$health" = 'yellow' ] || [ "$health" = 'green' ] ; do + health="$(curl -fsSL "$host/_cat/health?h=status")" + health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" + >&2 echo "Elasticsearch status is not green or yellow - sleeping" + sleep 10 +done + +>&2 echo "Elasticsearch is up" + + +exec $cmd diff --git a/docker/examples/mvt-tippecanoe/README.md b/docker/examples/mvt-tippecanoe/README.md new file mode 100644 index 0000000..b2ac906 --- /dev/null +++ b/docker/examples/mvt-tippecanoe/README.md @@ -0,0 +1,25 @@ +# pygeoapi with Tippecanoe MVT + +These folders contain a Docker Compose configuration necessary to setup a minimal +`pygeoapi` server that uses pre-rendered [Tippecanoe](https://github.com/mapbox/tippecanoe) Mapbox Vector Tiles to publish vector data as OGC API - Tiles. + +More information about this provider, including the features it supports, can be found on the [pygeoapi documentation](https://docs.pygeoapi.io/en/latest/data-publishing/ogcapi-tiles.html#providers#mvt-tippecanoe). + +This config is only for local development and testing. + +## Tippecanoe + +This example uses the test tiles, available on ```tests/data/tiles/ne_110m_lakes/```. You can generate your own tiles on disk with: + +``` bash +docker run -it --rm -v $(pwd)/data:/data emotionalcities/tippecanoe \ +tippecanoe --output-to-directory=/data/tiles/ --force --maximum-zoom=16 --drop-densest-as-needed --extend-zooms-if-still-dropping --no-tile-compression /data/ne_110m_populated_places_simple.geojson +``` + +## Building and Running + +To build and run the [Docker compose file](docker-compose.yml) in localhost: + +``` +docker-compose up +``` \ No newline at end of file diff --git a/docker/examples/mvt-tippecanoe/docker-compose.yml b/docker/examples/mvt-tippecanoe/docker-compose.yml new file mode 100644 index 0000000..0fbbdf9 --- /dev/null +++ b/docker/examples/mvt-tippecanoe/docker-compose.yml @@ -0,0 +1,41 @@ +# ================================================================= +# +# Authors: Just van den Broecke > +# Jorge Samuel Mendes de Jesus +# +# Copyright (c) 2019 Just van den Broecke +# Copyright (c) 2019 Jorge Samuel Mendes de Jesus +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +version: '3.3' + +services: + pygeoapi: + image: geopython/pygeoapi:latest + + container_name: pygeoapi + + ports: + - 5000:80 diff --git a/docker/examples/mvt-tippecanoe/pygeoapi/docker.config.yml b/docker/examples/mvt-tippecanoe/pygeoapi/docker.config.yml new file mode 100644 index 0000000..9c506ae --- /dev/null +++ b/docker/examples/mvt-tippecanoe/pygeoapi/docker.config.yml @@ -0,0 +1,124 @@ +# ================================================================= +# +# Authors: Joana Simoes +# +# Copyright (c) 2023 Joana Simoes +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +server: + bind: + host: 0.0.0.0 + port: 5000 + url: http://localhost:5000/ + mimetype: application/json; charset=UTF-8 + encoding: utf-8 + gzip: false + language: en-US + cors: true + pretty_print: true + limit: 10 + # templates: /path/to/templates + map: + url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png + attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' + +logging: + level: ERROR + #logfile: /tmp/pygeoapi.log + +metadata: + identification: + title: pygeoapi default instance + description: pygeoapi provides an API to geospatial data + keywords: + - geospatial + - data + - api + keywords_type: theme + terms_of_service: https://creativecommons.org/licenses/by/4.0/ + url: http://example.org + license: + name: CC-BY 4.0 license + url: https://creativecommons.org/licenses/by/4.0/ + provider: + name: Organization Name + url: https://pygeoapi.io + contact: + name: Lastname, Firstname + position: Position Title + address: Mailing Address + city: City + stateorprovince: Administrative Area + postalcode: Zip or Postal Code + country: Country + phone: +xx-xxx-xxx-xxxx + fax: +xx-xxx-xxx-xxxx + email: you@example.org + url: Contact URL + hours: Hours of Service + instructions: During hours of service. Off on weekends. + role: pointOfContact + +resources: + ne_110m_populated_places_simple: + type: collection + title: Populated Places + description: Point symbols with name attributes. Includes all admin-0 capitals and some other major cities. We favor regional significance over population census in determining our selection of places. Use the scale rankings to filter the number of towns that appear on your map. + keywords: + - populated places + - cities + - towns + links: + - type: text/html + rel: canonical + title: information + href: http://www.naturalearthdata.com/downloads/110m-cultural-vectors/110m-populated-places/ + hreflang: en-US + - type: application/gzip + rel: canonical + title: download + href: http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/110m/cultural/ne_110m_populated_places_simple.zip + hreflang: en-US + extents: + spatial: + bbox: [-180,-90,180,90] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: tile + name: MVT-tippecanoe + data: tests/data/tiles/ne_110m_lakes/ # local directory tree + # data: http://localhost:9000/ne_110m_lakes/{z}/{x}/{y}.pbf # tiles stored on a MinIO bucket + options: + metadata_format: default # default | tilejson + zoom: + min: 0 + max: 16 + schemes: + - WorldCRS84Quad + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + + diff --git a/docs/source/data-publishing/ogcapi-tiles.rst b/docs/source/data-publishing/ogcapi-tiles.rst index 29b33e8..3c0f74e 100644 --- a/docs/source/data-publishing/ogcapi-tiles.rst +++ b/docs/source/data-publishing/ogcapi-tiles.rst @@ -7,50 +7,42 @@ Publishing tiles to OGC API - Tiles (map, vector, coverage, etc.). pygeoapi can publish tiles from local or remote data sources (including cloud -object storage or a tile service). To integrate tiles from a local data source, it is assumed -that a directory tree of static tiles has been created on disk. Examples of -tile generation software include (but are not limited to): - -* `MapProxy`_ -* `tippecanoe`_ +object storage or a tile service). Providers --------- -pygeoapi core tile providers are listed below, along with supported storage types. +pygeoapi core tile providers are listed below, along with supported features. .. csv-table:: - :header: Provider, local, remote + :header: Provider, rendered on-the-fly, properties :align: left - `MVT`_,✅,✅ - + `MVT-tippecanoe`_,❌,✅ + `MVT-elastic`_,✅,❌ Below are specific connection examples based on supported providers. +.. note:: + Currently only `Mapbox Vector Tiles (MVT) `_ are supported in pygeoapi. + Connection examples ------------------- -MVT -^^^ +MVT-tippecanoe +^^^^^^^^^^^^^^ -The MVT provider plugin provides access to `Mapbox Vector Tiles`_. +This provider gives support to serving tiles generated using `Mapbox Tippecanoe `_. +The tiles can be integrated from a path on disk, or from a static url (e.g.: from an S3 or MinIO bucket). +In both cases, they have to be rendered before using pygeoapi. -Remote data sources can be any external service (i.e. Elasticsearch), by providing a URL -template. - -.. note:: - Currently, the URL templating in this provider supports the following formats: `/{z}/{x}/{y}` or `/{z}/{y}/{x}`. - For additional formats: feel free to file an `issue `_. - - -This code block shows how to configure pygeoapi to read Mapbox vector tiles, from disk or a URL. +This code block shows how to configure pygeoapi to read Mapbox vector tiles generated with tippecanoe, from disk or a URL. .. code-block:: yaml providers: - type: tile - name: MVT + name: MVT-tippecanoe data: tests/data/tiles/ne_110m_lakes # local directory tree # data: http://localhost:9000/ne_110m_lakes/{z}/{x}/{y}.pbf # tiles stored on a MinIO bucket options: @@ -64,13 +56,23 @@ This code block shows how to configure pygeoapi to read Mapbox vector tiles, fro name: pbf mimetype: application/vnd.mapbox-vector-tile -This code block shows how to configure pygeoapi to read Mapbox vector tiles, from an `Elasticsearch `_ endpoint. +.. tip:: + On `this tutorial `_ you can find detailed instructions on how-to generate tiles using tippecanoe and integrate them into pygeoapi. + +MVT-elastic +^^^^^^^^^^^^ + +This provider gives support to serving tiles generated using `Elasticsearch `_. +These tiles are rendered on-the-fly using the `Elasticsearch Vector tile search API `_. +In order to use it, the only requirement is to have the data stored in an Elasticsearch index. + +This code block shows how to configure pygeoapi to read Mapbox vector tiles from an Elasticsearch endpoint. .. code-block:: yaml providers: - type: tile - name: MVT + name: MVT-elastic data: http://localhost:9200/ne_110m_populated_places_simple2/_mvt/geometry/{z}/{x}/{y}?grid_precision=0 # if you don't use precision 0, you will be requesting for aggregations which are not supported in the # free version of elastic @@ -85,25 +87,9 @@ This code block shows how to configure pygeoapi to read Mapbox vector tiles, fro name: pbf mimetype: application/vnd.mapbox-vector-tile -This code block shows how to configure pygeoapi to read Mapbox vector tiles, from a `pg_tileserv `_ endpoint. - -.. code-block:: yaml - - providers: - - type: tile - name: MVT - data: http://localhost:7800/public.ne_50m_admin_0_countries/{z}/{x}/{y}.pbf - options: - metadata_format: default # default | tilejson - zoom: - min: 0 - max: 16 - schemes: - - WorldCRS84Quad - format: - name: pbf - mimetype: application/vnd.mapbox-vector-tile - +.. tip:: + On `this tutorial `_ you can find detailed instructions on publish tiles stored in an Elasticsearch endpoint. + Data access examples -------------------- @@ -120,6 +106,6 @@ Data access examples .. _`OGC API - Tiles`: https://github.com/opengeospatial/ogcapi-tiles -.. _`MapProxy`: https://mapproxy.org .. _`tippecanoe`: https://github.com/mapbox/tippecanoe +.. _`Elasticsearch`: https://www.elastic.co/ .. _`Mapbox Vector Tiles`: https://docs.mapbox.com/data/tilesets/guides/vector-tiles-introduction/ diff --git a/pygeoapi/plugin.py b/pygeoapi/plugin.py index 9608ca3..312017a 100644 --- a/pygeoapi/plugin.py +++ b/pygeoapi/plugin.py @@ -50,7 +50,8 @@ PLUGINS = { 'Hateoas': 'pygeoapi.provider.hateoas.HateoasProvider', 'MapScript': 'pygeoapi.provider.mapscript_.MapScriptProvider', 'MongoDB': 'pygeoapi.provider.mongo.MongoProvider', - 'MVT': 'pygeoapi.provider.mvt.MVTProvider', + 'MVT-tippecanoe': 'pygeoapi.provider.mvt_tippecanoe.MVTTippecanoeProvider', # noqa: E501 + 'MVT-elastic': 'pygeoapi.provider.mvt_elastic.MVTElasticProvider', # noqa: E501 'OracleDB': 'pygeoapi.provider.oracle.OracleProvider', 'OGR': 'pygeoapi.provider.ogr.OGRProvider', 'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider', diff --git a/pygeoapi/provider/mvt.py b/pygeoapi/provider/base_mvt.py similarity index 70% rename from pygeoapi/provider/mvt.py rename to pygeoapi/provider/base_mvt.py index 6a7c64a..796d9c4 100644 --- a/pygeoapi/provider/mvt.py +++ b/pygeoapi/provider/base_mvt.py @@ -35,8 +35,7 @@ import requests from pathlib import Path from urllib.parse import urlparse -from pygeoapi.provider.tile import ( - BaseTileProvider, ProviderTileNotFoundError) +from pygeoapi.provider.tile import BaseTileProvider from pygeoapi.provider.base import ProviderConnectionError from pygeoapi.models.provider.base import ( TileMatrixSetEnum, TilesMetadataFormat, TileSetMetadata, LinkType, @@ -47,8 +46,8 @@ from pygeoapi.util import is_url, url_join LOGGER = logging.getLogger(__name__) -class MVTProvider(BaseTileProvider): - """MVT Provider""" +class BaseMVTProvider(BaseTileProvider): + """Base MVT Provider""" def __init__(self, provider_def): """ @@ -56,46 +55,15 @@ class MVTProvider(BaseTileProvider): :param provider_def: provider definition - :returns: pygeoapi.provider.MVT.MVTProvider + :returns: pygeoapi.provider.base_mvt.BaseMVTProvider """ super().__init__(provider_def) self.tile_type = 'vector' - if is_url(self.data): - url = urlparse(self.data) - baseurl = f'{url.scheme}://{url.netloc}' - param_type = '?f=mvt' - layer = f'/{self.get_layer()}' - - LOGGER.debug('Extracting layer name from URL') - LOGGER.debug(f'Layer: {layer}') - - tilepath = f'{layer}/tiles' - servicepath = f'{tilepath}/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}{param_type}' # noqa - - self._service_url = url_join(baseurl, servicepath) - - self._service_metadata_url = url_join( - self.service_url.split('{tileMatrix}/{tileRow}/{tileCol}')[0], - 'metadata') - else: - data_path = Path(self.data) - if not data_path.exists(): - msg = f'Service does not exist: {self.data}' - LOGGER.error(msg) - raise ProviderConnectionError(msg) - self._service_url = data_path - metadata_path = data_path.joinpath('metadata.json') - if not metadata_path.exists(): - msg = f'Service metadata does not exist: {metadata_path.name}' - LOGGER.error(msg) - LOGGER.warning(msg) - self._service_metadata_url = metadata_path - def __repr__(self): - return f' {self.data}' + raise NotImplementedError() @property def service_url(self): @@ -106,27 +74,7 @@ class MVTProvider(BaseTileProvider): return self._service_metadata_url def get_layer(self): - - if is_url(self.data): - url = urlparse(self.data) - # We need to try, at least these different variations that - # I have seen across products (maybe there more??) - - if ('/{z}/{x}/{y}' not in url.path and - '/{z}/{y}/{x}' not in url.path): - msg = f'This url template is not supported yet: {url.path}' - LOGGER.error(msg) - raise ProviderConnectionError(msg) - - layer = url.path.split('/{z}/{x}/{y}')[0] - layer = layer.split('/{z}/{y}/{x}')[0] - - LOGGER.debug(layer) - LOGGER.debug('Removing leading "/"') - return layer[1:] - - else: - return Path(self.data).name + raise NotImplementedError() def get_tiling_schemes(self): @@ -160,36 +108,6 @@ class MVTProvider(BaseTileProvider): basepath = url.path.split('/{z}/{x}/{y}')[0] servicepath = servicepath or f'{basepath}/tiles/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}{tile_type}' # noqa - if servicepath.startswith(baseurl): - self._service_url = servicepath - else: - self._service_url = url_join(baseurl, servicepath) - tile_matrix_set = self.service_url.split( - '/{tileMatrix}/{tileRow}/{tileCol}')[0] - self._service_metadata_url = url_join(tile_matrix_set, 'metadata') - links = { - 'links': [ - { - 'type': 'application/json', - 'rel': 'self', - 'title': 'This collection as multi vector tilesets', - 'href': f'{tile_matrix_set}?f=json' - }, - { - 'type': self.mimetype, - 'rel': 'item', - 'title': 'This collection as multi vector tiles', - 'href': self.service_url - }, { - 'type': 'application/json', - 'rel': 'describedby', - 'title': 'Collection metadata in TileJSON format', - 'href': f'{self.service_metadata_url}?f=json' - } - ] - } - return links - def get_tiles(self, layer=None, tileset=None, z=None, y=None, x=None, format_=None): """ @@ -204,39 +122,8 @@ class MVTProvider(BaseTileProvider): :returns: an encoded mvt tile """ - if format_ == "mvt": - format_ = self.format_type - if is_url(self.data): - url = urlparse(self.data) - base_url = f'{url.scheme}://{url.netloc}' - if url.query: - url_query = f'?{url.query}' - else: - url_query = '' - - with requests.Session() as session: - session.get(base_url) - # There is a "." in the url path - if '.' in url.path: - resp = session.get(f'{base_url}/{layer}/{z}/{y}/{x}.{format_}{url_query}') # noqa - # There is no "." in the url )e.g. elasticsearch) - else: - resp = session.get(f'{base_url}/{layer}/{z}/{y}/{x}{url_query}') # noqa - resp.raise_for_status() - return resp.content - else: - if not isinstance(self.service_url, Path): - msg = f'Wrong data path configuration: {self.service_url}' - LOGGER.error(msg) - raise ProviderConnectionError(msg) - else: - try: - service_url_path = self.service_url.joinpath(f'{z}/{y}/{x}.{format_}') # noqa - with open(service_url_path, mode='rb') as tile: - return tile.read() - except FileNotFoundError as err: - raise ProviderTileNotFoundError(err) + raise NotImplementedError() def get_metadata(self, dataset, server_url, layer=None, tileset=None, metadata_format=None, title=None, @@ -332,3 +219,36 @@ class MVTProvider(BaseTileProvider): layers.append(GeospatialDataType(id=vector_layer['id'])) content.layers = layers return content.dict(exclude_none=True) + + def get_tms_links(self): + """ + Generates TileMatrixSet Links + + :returns: a JSON object with TMS links + """ + + tile_matrix_set = self.service_url.split( + '/{tileMatrix}/{tileRow}/{tileCol}')[0] + self._service_metadata_url = url_join(tile_matrix_set, 'metadata') + links = { + 'links': [ + { + 'type': 'application/json', + 'rel': 'self', + 'title': 'This collection as multi vector tilesets', + 'href': f'{tile_matrix_set}?f=json' + }, + { + 'type': self.mimetype, + 'rel': 'item', + 'title': 'This collection as multi vector tiles', + 'href': self.service_url + }, { + 'type': 'application/json', + 'rel': 'describedby', + 'title': 'Collection metadata in TileJSON format', + 'href': f'{self.service_metadata_url}?f=json' + } + ] + } + return links diff --git a/pygeoapi/provider/mvt_elastic.py b/pygeoapi/provider/mvt_elastic.py new file mode 100644 index 0000000..626742f --- /dev/null +++ b/pygeoapi/provider/mvt_elastic.py @@ -0,0 +1,191 @@ +# ================================================================= +# +# Authors: Joana Simoes +# +# Copyright (c) 2023 Joana Simoes +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging +import requests +from urllib.parse import urlparse + +from pygeoapi.provider.base_mvt import BaseMVTProvider +from pygeoapi.provider.base import ProviderConnectionError +from pygeoapi.util import is_url, url_join + +LOGGER = logging.getLogger(__name__) + + +class MVTElasticProvider(BaseMVTProvider): + """MVT Elastic Provider + Provider for serving tiles rendered with the Elasticsearch + Vector Tile API + https://www.elastic.co/guide/en/elasticsearch/reference/current/search-vector-tile-api.html + As of 12/23, elastic does not provide any tileset metadata. + """ + + def __init__(self, BaseMVTProvider): + """ + Initialize object + + :param provider_def: provider definition + + :returns: pygeoapi.provider.MVT.MVTElasticProvider + """ + + super().__init__(BaseMVTProvider) + + if not is_url(self.data): + msg = 'Wrong input format for Elasticsearch MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + url = urlparse(self.data) + baseurl = f'{url.scheme}://{url.netloc}' + param_type = '?f=mvt' + layer = f'/{self.get_layer()}' + + LOGGER.debug('Extracting layer name from URL') + LOGGER.debug(f'Layer: {layer}') + + tilepath = f'{layer}/tiles' + servicepath = f'{tilepath}/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}{param_type}' # noqa + + self._service_url = url_join(baseurl, servicepath) + + self._service_metadata_url = url_join( + self.service_url.split('{tileMatrix}/{tileRow}/{tileCol}')[0], + 'metadata') + + def __repr__(self): + return f' {self.data}' + + @property + def service_url(self): + return self._service_url + + @property + def service_metadata_url(self): + return self._service_metadata_url + + def get_layer(self): + """ + Extracts layer name from url + + :returns: layer name + """ + + if not is_url(self.data): + msg = 'Wrong input format for Elasticsearch MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + url = urlparse(self.data) + + if ('/{z}/{x}/{y}' not in url.path): + msg = 'Wrong input format for Elasticsearch MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + layer = url.path.split('/{z}/{x}/{y}')[0] + + LOGGER.debug(layer) + LOGGER.debug('Removing leading "/"') + return layer[1:] + + def get_tiles_service(self, baseurl=None, servicepath=None, + dirpath=None, tile_type=None): + """ + Gets mvt service description + + :param baseurl: base URL of endpoint + :param servicepath: base path of URL + :param dirpath: directory basepath (equivalent of URL) + :param tile_type: tile format type + + :returns: `dict` of item tile service + """ + + super().get_tiles_service(baseurl, servicepath, + dirpath, tile_type) + + self._service_url = servicepath + return self.get_tms_links() + + def get_tiles(self, layer=None, tileset=None, + z=None, y=None, x=None, format_=None): + """ + Gets tile + + :param layer: mvt tile layer + :param tileset: mvt tileset + :param z: z index + :param y: y index + :param x: x index + :param format_: tile format + + :returns: an encoded mvt tile + """ + if format_ == 'mvt': + format_ = self.format_type + + if is_url(self.data): + url = urlparse(self.data) + base_url = f'{url.scheme}://{url.netloc}' + + if url.query: + url_query = f'?{url.query}' + else: + url_query = '' + + with requests.Session() as session: + session.get(base_url) + resp = session.get(f'{base_url}/{layer}/{z}/{y}/{x}{url_query}') # noqa + resp.raise_for_status() + return resp.content + else: + msg = 'Wrong input format for Elasticsearch MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + def get_metadata(self, dataset, server_url, layer=None, + tileset=None, metadata_format=None, title=None, + description=None, keywords=None, **kwargs): + """ + Gets tile metadata + + :param dataset: dataset name + :param server_url: server base url + :param layer: mvt tile layer name + :param tileset: mvt tileset name + :param metadata_format: format for metadata, + enum TilesMetadataFormat + + :returns: `dict` of JSON metadata + """ + + return super().get_metadata(dataset, server_url, layer, + tileset, metadata_format, title, + description, keywords, **kwargs) diff --git a/pygeoapi/provider/mvt_tippecanoe.py b/pygeoapi/provider/mvt_tippecanoe.py new file mode 100644 index 0000000..88b315c --- /dev/null +++ b/pygeoapi/provider/mvt_tippecanoe.py @@ -0,0 +1,190 @@ +# ================================================================= +# +# Authors: Joana Simoes +# +# Copyright (c) 2023 Joana Simoes +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging +from pathlib import Path +from urllib.parse import urlparse + +from pygeoapi.provider.tile import ( + ProviderTileNotFoundError) +from pygeoapi.provider.base_mvt import BaseMVTProvider +from pygeoapi.provider.base import ProviderConnectionError +from pygeoapi.util import is_url, url_join + +LOGGER = logging.getLogger(__name__) + + +class MVTTippecanoeProvider(BaseMVTProvider): + """MVT Tippecanoe Provider + Provider for serving tiles generated with Mapbox Tippecanoe + https://github.com/mapbox/tippecanoe + It supports both, tiles from a an url or a path on disk. + Tippecanoe also provides a TileSet Metadata in a file called + "metadata.json". + """ + + def __init__(self, BaseMVTProvider): + """ + Initialize object + + :param provider_def: provider definition + + :returns: pygeoapi.provider.MVT.MVTTippecanoeProvider + """ + + super().__init__(BaseMVTProvider) + + # Pre-rendered tiles served from a static url + if is_url(self.data): + url = urlparse(self.data) + baseurl = f'{url.scheme}://{url.netloc}' + param_type = '?f=mvt' + layer = f'/{self.get_layer()}' + + LOGGER.debug('Extracting layer name from URL') + LOGGER.debug(f'Layer: {layer}') + + tilepath = f'{layer}/tiles' + servicepath = f'{tilepath}/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}{param_type}' # noqa + + self._service_url = url_join(baseurl, servicepath) + + self._service_metadata_url = url_join( + self.service_url.split('{tileMatrix}/{tileRow}/{tileCol}')[0], + 'metadata') + # Pre-rendered tiles served from a local path + else: + data_path = Path(self.data) + if not data_path.exists(): + msg = f'Service does not exist: {self.data}' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + self._service_url = data_path + metadata_path = data_path.joinpath('metadata.json') + if not metadata_path.exists(): + msg = f'Service metadata does not exist: {metadata_path.name}' + LOGGER.error(msg) + LOGGER.warning(msg) + self._service_metadata_url = metadata_path + + def __repr__(self): + return f' {self.data}' + + def get_layer(self): + """ + Extracts layer name from url or data path + + :returns: layer name + """ + + if is_url(self.data): + url = urlparse(self.data) + + if ('/{z}/{x}/{y}' not in url.path): + msg = f'This url template is not supported yet: {url.path}' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + layer = url.path.split('/{z}/{x}/{y}')[0] + + LOGGER.debug(layer) + LOGGER.debug('Removing leading "/"') + return layer[1:] + + else: + return Path(self.data).name + + def get_tiles_service(self, baseurl=None, servicepath=None, + dirpath=None, tile_type=None): + """ + Gets mvt service description + + :param baseurl: base URL of endpoint + :param servicepath: base path of URL + :param dirpath: directory basepath (equivalent of URL) + :param tile_type: tile format type + + :returns: `dict` of item tile service + """ + + super().get_tiles_service(baseurl, servicepath, + dirpath, tile_type) + + self._service_url = servicepath + return self.get_tms_links() + + def get_tiles(self, layer=None, tileset=None, + z=None, y=None, x=None, format_=None): + """ + Gets tile + + :param layer: mvt tile layer + :param tileset: mvt tileset + :param z: z index + :param y: y index + :param x: x index + :param format_: tile format + + :returns: an encoded mvt tile + """ + + if format_ == 'mvt': + format_ = self.format_type + + if not isinstance(self.service_url, Path): + msg = f'Wrong data path configuration: {self.service_url}' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + else: + try: + service_url_path = self.service_url.joinpath(f'{z}/{y}/{x}.{format_}') # noqa + with open(service_url_path, mode='rb') as tile: + return tile.read() + except FileNotFoundError as err: + raise ProviderTileNotFoundError(err) + + def get_metadata(self, dataset, server_url, layer=None, + tileset=None, metadata_format=None, title=None, + description=None, keywords=None, **kwargs): + """ + Gets tile metadata + + :param dataset: dataset name + :param server_url: server base url + :param layer: mvt tile layer name + :param tileset: mvt tileset name + :param metadata_format: format for metadata, + enum TilesMetadataFormat + + :returns: `dict` of JSON metadata + """ + + return super().get_metadata(dataset, server_url, layer, + tileset, metadata_format, title, + description, keywords, **kwargs) diff --git a/tests/pygeoapi-test-config.yml b/tests/pygeoapi-test-config.yml index 2cd2579..e085689 100644 --- a/tests/pygeoapi-test-config.yml +++ b/tests/pygeoapi-test-config.yml @@ -246,11 +246,11 @@ resources: storage_crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 storage_crs_coordinate_epoch: 2017.23 - type: tile - name: MVT + name: MVT-tippecanoe # data: http://localhost:9000/ne_110m_lakes/{z}/{x}/{y} data: tests/data/tiles/ne_110m_lakes options: - metadata_format: raw # default | tilejson + metadata_format: default # default | tilejson bounds: [[-124.953634,-16.536406],[109.929807,66.969298]] zoom: min: 0