From 85a4984bbed349c01ec40550bbd0ac8dd2c371f0 Mon Sep 17 00:00:00 2001 From: Jo Date: Thu, 1 Dec 2022 14:34:16 +0100 Subject: [PATCH] Add Support for a generic url template, for publishing vector tiles as OGC API Tiles (#1050) * - Added support to read from a generic url template as vector tile backend. * - Add support for not rendering the tileset metadata * - added support for z/y/x vector tile layers * - fixed formatting issues * - refactored code to use get_layer function, everywhere we need to parse username - added a couple of debug statements, to make sure we are getting the layer name correctly - added an error for url templates that follow a schema, which is not supported yet * - Added documentation for tiles provider, to show how to read a generic url * - fixed formatting issues * - added example of ES vector tiles in the docker-config of elasticsearch Co-authored-by: doublebyte1 --- .../elastic/pygeoapi/docker.config.yml | 15 ++++++ docs/source/data-publishing/ogcapi-tiles.rst | 32 +++++++++++- pygeoapi/provider/mvt.py | 50 ++++++++++++++++--- .../templates/collections/tiles/index.html | 13 +++-- 4 files changed, 97 insertions(+), 13 deletions(-) diff --git a/docker/examples/elastic/pygeoapi/docker.config.yml b/docker/examples/elastic/pygeoapi/docker.config.yml index 428c77e..b909fb3 100644 --- a/docker/examples/elastic/pygeoapi/docker.config.yml +++ b/docker/examples/elastic/pygeoapi/docker.config.yml @@ -150,6 +150,21 @@ resources: #Note elastic_search is the docker container of ES the name is defined in the docker-compose.yml data: http://elastic_search:9200/ne_110m_populated_places_simple id_field: geonameid + - type: tile + name: MVT + data: http://elastic_search:9200/ne_110m_populated_places_simple/_mvt/geometry/{z}/{x}/{y}?grid_precision=0 + # index must have a geo_point + options: + metadata_format: none # default | tilejson + zoom: + min: 0 + max: 16 + schemes: + - WorldCRS84Quad + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + lakes: type: collection diff --git a/docs/source/data-publishing/ogcapi-tiles.rst b/docs/source/data-publishing/ogcapi-tiles.rst index 25df29a..33f1689 100644 --- a/docs/source/data-publishing/ogcapi-tiles.rst +++ b/docs/source/data-publishing/ogcapi-tiles.rst @@ -7,13 +7,19 @@ Publishing map tiles to OGC API - Tiles (map, vector, etc.). pygeoapi can publish tiles from local or remote data sources (including cloud -object storage). To integrate tiles from a local data source, it is assumed +object storage or a tile service). To integrate tiles from a local data source, it is assumed that a directory tree of static tiles has been created on disk. Examples of tile generation software include (but are not limited to): * `MapProxy`_ * `tippecanoe`_ +The remote data sources can be an external service like Elasticsearch, read from a generic url template. + +.. note:: + Currently, the url template only supports the formats: `/{z}/{x}/{y}` or `/{z}/{y}/{x}`. + If you have a different use case, feel free to file an `issue `_. + Providers --------- @@ -36,13 +42,15 @@ MVT The MVT provider plugin provides access to `Mapbox Vector Tiles`_. +This code block shows how to configure pygeoapi to read Mapbox vector tiles, from disk or from an url. + .. code-block:: yaml providers: - type: tile name: MVT data: tests/data/tiles/ne_110m_lakes # local directory tree - # data: https://example.org/ne_110m_lakes/{z}/{x}/{y}.pbf + # data: http://localhost:9000/ne_110m_lakes/{z}/{x}/{y}.pbf # tiles stored on a Minio bucket options: metadata_format: raw # default | tilejson zoom: @@ -54,6 +62,26 @@ The MVT provider plugin provides access to `Mapbox Vector Tiles`_. name: pbf mimetype: application/vnd.mapbox-vector-tile +This code block shows how to configure pygeoapi to read Mapbox vector tiles, from an Elasticsearch endpoint. + +.. code-block:: yaml + + providers: + - type: tile + name: MVT + data: http://localhost:9200/ne_110m_populated_places_simple2/_mvt/geometry/{z}/{x}/{y}?grid_precision=0 + # if you don't use precision 0, you will be requesting for aggregations which are not supported in the + # free version of elastic + options: + metadata_format: raw # default | tilejson + zoom: + min: 0 + max: 5 + schemes: + - WorldCRS84Quad + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile Data access examples -------------------- diff --git a/pygeoapi/provider/mvt.py b/pygeoapi/provider/mvt.py index 65388eb..61e8303 100644 --- a/pygeoapi/provider/mvt.py +++ b/pygeoapi/provider/mvt.py @@ -59,7 +59,11 @@ class MVTProvider(BaseTileProvider): url = urlparse(self.data) baseurl = '{}://{}'.format(url.scheme, url.netloc) param_type = '?f=mvt' - layer = url.path.split('/{z}/{x}/{y}')[0] + layer = '/' + self.get_layer() + + LOGGER.debug('Extracting layer name from url') + LOGGER.debug('Layer: {}'.format(layer)) + tilepath = '{}/tiles'.format(layer) servicepath = \ '{}/{{{}}}/{{{}}}/{{{}}}/{{{}}}{}'.format( @@ -68,9 +72,11 @@ class MVTProvider(BaseTileProvider): 'tileMatrix', 'tileRow', 'tileCol', - param_type) + param_type + ) self._service_url = url_join(baseurl, servicepath) + self._service_metadata_url = urljoin( self.service_url.split('{tileMatrix}/{tileRow}/{tileCol}')[0], 'metadata') @@ -104,7 +110,28 @@ class MVTProvider(BaseTileProvider): if is_url(self.data): url = urlparse(self.data) - return url.path.split("/{z}/{x}/{y}")[0][1:] + # We need to try, at least these different variations that + # I have seen across products (maybe there more??) + + if ("/{z}/{x}/{y}" not in url.path and + "/{z}/{y}/{x}" not in url.path): + msg = 'This url template is not supported yet: {}'.format( + url.path) + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + layer = url.path.split("/{z}/{x}/{y}")[0] + layer = layer.split("/{z}/{y}/{x}")[0] + + # Removing the extension, if it is there + if ('.' in layer): + layer = layer.split('.')[0] + + LOGGER.debug(layer) + # Removing the first "/" + layer = layer[1:] + LOGGER.debug(layer) + return layer else: return Path(self.data).name @@ -204,9 +231,20 @@ class MVTProvider(BaseTileProvider): base_url = '{}://{}'.format(url.scheme, url.netloc) with requests.Session() as session: session.get(base_url) - resp = session.get('{base_url}/{lyr}/{z}/{y}/{x}.{f}'.format( - base_url=base_url, lyr=layer, - z=z, y=y, x=x, f=format_)) + # There is a "." in the url path + if '.' in url.path: + resp = session.get( + '{base_url}/{lyr}/{z}/{y}/{x}.{f}{q}'.format( + base_url=base_url, lyr=layer, + z=z, y=y, x=x, f=format_, q="?" + url.query + if url.query else '')) + # There is no "." in the url )e.g. elasticsearch) + else: + resp = session.get( + '{base_url}/{lyr}/{z}/{y}/{x}{q}'.format( + base_url=base_url, lyr=layer, + z=z, y=y, x=x, q="?" + url.query + if url.query else '')) resp.raise_for_status() return resp.content else: diff --git a/pygeoapi/templates/collections/tiles/index.html b/pygeoapi/templates/collections/tiles/index.html index 1a4bea4..c1f0c53 100644 --- a/pygeoapi/templates/collections/tiles/index.html +++ b/pygeoapi/templates/collections/tiles/index.html @@ -39,9 +39,10 @@
-
{% trans %}Metadata{% endtrans %}
-
Tiles metadata in {{ data['format'] }} format
-
+ {% if data['format']=='tilejson' %} +
{% trans %}Metadata{% endtrans %}
+
Tiles metadata in {{ data['format'] }} format
+ {% endif %} + if (document.getElementById("tilejson")){ + document.getElementById("tilejson").href = "{{ config['server']['url'] }}/collections/{{ data['id'] }}/tiles/" + tileset + "/metadata"; + } +
Map