Add Support for a generic url template, for publishing vector tiles as OGC API Tiles (#1050)

* - Added support to read from a generic url template as vector tile backend.

* - Add support for not rendering the tileset metadata

* - added support for z/y/x vector tile layers

* - fixed formatting issues

* - refactored code to use get_layer function, everywhere we need to parse username
- added a couple of debug statements, to make sure we are getting the layer name
correctly
- added an error for url templates that follow a schema, which is not supported yet

* - Added documentation for tiles provider, to show how to read a generic url

* - fixed formatting issues

* - added example of ES vector tiles in the docker-config of elasticsearch

Co-authored-by: doublebyte1 <info@doublebyte.net>

docker/examples/elastic/pygeoapi/docker.config.yml

@@ -150,6 +150,21 @@ resources:
               #Note elastic_search is the docker container of ES the name is defined in the docker-compose.yml
               data: http://elastic_search:9200/ne_110m_populated_places_simple
               id_field: geonameid
+            - type: tile
+              name: MVT
+              data: http://elastic_search:9200/ne_110m_populated_places_simple/_mvt/geometry/{z}/{x}/{y}?grid_precision=0
+              # index must have a geo_point
+              options:
+                metadata_format: none # default | tilejson
+                zoom:
+                    min: 0
+                    max: 16
+                schemes:
+                    - WorldCRS84Quad
+              format:
+                    name: pbf
+                    mimetype: application/vnd.mapbox-vector-tile
+
 
     lakes:
         type: collection

docs/source/data-publishing/ogcapi-tiles.rst

@@ -7,13 +7,19 @@ Publishing map tiles to OGC API - Tiles
 (map, vector, etc.).
 
 pygeoapi can publish tiles from local or remote data sources (including cloud
-object storage).  To integrate tiles from a local data source, it is assumed
+object storage or a tile service). To integrate tiles from a local data source, it is assumed
 that a directory tree of static tiles has been created on disk.  Examples of
 tile generation software include (but are not limited to):
 
 * `MapProxy`_
 * `tippecanoe`_
 
+The remote data sources can be an external service like Elasticsearch, read from a generic url template.
+
+.. note::
+   Currently, the url template only supports the formats: `/{z}/{x}/{y}` or `/{z}/{y}/{x}`. 
+   If you have a different use case, feel free to file an `issue <https://github.com/geopython/pygeoapi/issues>`_.  
+
 Providers
 ---------
 
@@ -36,13 +42,15 @@ MVT
 
 The MVT provider plugin provides access to `Mapbox Vector Tiles`_.
 
+This code block shows how to configure pygeoapi to read Mapbox vector tiles, from disk or from an url.
+
 .. code-block:: yaml
 
    providers:
        - type: tile
          name: MVT 
          data: tests/data/tiles/ne_110m_lakes  # local directory tree
-         # data: https://example.org/ne_110m_lakes/{z}/{x}/{y}.pbf
+         # data: http://localhost:9000/ne_110m_lakes/{z}/{x}/{y}.pbf # tiles stored on a Minio bucket
          options:
              metadata_format: raw # default | tilejson
              zoom:
@@ -54,6 +62,26 @@ The MVT provider plugin provides access to `Mapbox Vector Tiles`_.
              name: pbf 
              mimetype: application/vnd.mapbox-vector-tile
 
+This code block shows how to configure pygeoapi to read Mapbox vector tiles, from an Elasticsearch endpoint.
+
+.. code-block:: yaml
+
+   providers:
+       - type: tile
+         name: MVT 
+         data: http://localhost:9200/ne_110m_populated_places_simple2/_mvt/geometry/{z}/{x}/{y}?grid_precision=0
+         # if you don't use precision 0, you will be requesting for aggregations which are not supported in the 
+         # free version of elastic
+         options:
+             metadata_format: raw # default | tilejson
+             zoom:
+                 min: 0
+                 max: 5
+             schemes:
+                 - WorldCRS84Quad
+         format:
+             name: pbf 
+             mimetype: application/vnd.mapbox-vector-tile
 
 Data access examples
 --------------------

pygeoapi/provider/mvt.py

@@ -59,7 +59,11 @@ class MVTProvider(BaseTileProvider):
             url = urlparse(self.data)
             baseurl = '{}://{}'.format(url.scheme, url.netloc)
             param_type = '?f=mvt'
-            layer = url.path.split('/{z}/{x}/{y}')[0]
+            layer = '/' + self.get_layer()
+
+            LOGGER.debug('Extracting layer name from url')
+            LOGGER.debug('Layer: {}'.format(layer))
+
             tilepath = '{}/tiles'.format(layer)
             servicepath = \
                 '{}/{{{}}}/{{{}}}/{{{}}}/{{{}}}{}'.format(
@@ -68,9 +72,11 @@ class MVTProvider(BaseTileProvider):
                     'tileMatrix',
                     'tileRow',
                     'tileCol',
-                    param_type)
+                    param_type
+                    )
 
             self._service_url = url_join(baseurl, servicepath)
+
             self._service_metadata_url = urljoin(
                 self.service_url.split('{tileMatrix}/{tileRow}/{tileCol}')[0],
                 'metadata')
@@ -104,7 +110,28 @@ class MVTProvider(BaseTileProvider):
 
         if is_url(self.data):
             url = urlparse(self.data)
-            return url.path.split("/{z}/{x}/{y}")[0][1:]
+            # We need to try, at least these different variations that
+            # I have seen across products (maybe there more??)
+
+            if ("/{z}/{x}/{y}" not in url.path and
+                    "/{z}/{y}/{x}" not in url.path):
+                msg = 'This url template is not supported yet: {}'.format(
+                    url.path)
+                LOGGER.error(msg)
+                raise ProviderConnectionError(msg)
+
+            layer = url.path.split("/{z}/{x}/{y}")[0]
+            layer = layer.split("/{z}/{y}/{x}")[0]
+
+            # Removing the extension, if it is there
+            if ('.' in layer):
+                layer = layer.split('.')[0]
+
+            LOGGER.debug(layer)
+            # Removing the first "/"
+            layer = layer[1:]
+            LOGGER.debug(layer)
+            return layer
         else:
             return Path(self.data).name
 
@@ -204,9 +231,20 @@ class MVTProvider(BaseTileProvider):
             base_url = '{}://{}'.format(url.scheme, url.netloc)
             with requests.Session() as session:
                 session.get(base_url)
-                resp = session.get('{base_url}/{lyr}/{z}/{y}/{x}.{f}'.format(
-                    base_url=base_url, lyr=layer,
-                    z=z, y=y, x=x, f=format_))
+                # There is a "." in the url path
+                if '.' in url.path:
+                    resp = session.get(
+                        '{base_url}/{lyr}/{z}/{y}/{x}.{f}{q}'.format(
+                            base_url=base_url, lyr=layer,
+                            z=z, y=y, x=x, f=format_, q="?" + url.query
+                            if url.query else ''))
+                # There is no "." in the url )e.g. elasticsearch)
+                else:
+                    resp = session.get(
+                        '{base_url}/{lyr}/{z}/{y}/{x}{q}'.format(
+                            base_url=base_url, lyr=layer,
+                            z=z, y=y, x=x, q="?" + url.query
+                            if url.query else ''))
                 resp.raise_for_status()
                 return resp.content
         else:

pygeoapi/templates/collections/tiles/index.html

@@ -39,9 +39,10 @@
       </div>
       <br/>
       <div class="row">
-        <div class="col-md-2 col-sm-12">{% trans %}Metadata{% endtrans %}</div>
-        <div class="col-md-8"><a id="tilejson" href="" target="_blank">Tiles metadata in {{ data['format'] }} format</a></div>
-      </div>
+        {% if data['format']=='tilejson' %}
+          <div class="col-md-2 col-sm-12">{% trans %}Metadata{% endtrans %}</div>
+          <div class="col-md-8"><a id="tilejson" href="" target="_blank">Tiles metadata in {{ data['format'] }} format</a></div>
+        {% endif %}      </div>
       <script>
         var select = document.getElementById('tilingScheme');
         var tileset = select.value;
@@ -54,8 +55,10 @@
           var scheme = ev.target.value;
           document.location.search = `scheme=${scheme}`;
         });
-        document.getElementById("tilejson").href = "{{ config['server']['url'] }}/collections/{{ data['id'] }}/tiles/" + tileset + "/metadata";
-      </script>
+        if (document.getElementById("tilejson")){
+          document.getElementById("tilejson").href = "{{ config['server']['url'] }}/collections/{{ data['id'] }}/tiles/" + tileset + "/metadata";
+        }
+        </script>
       <br/>
       <div class="row">
         <div class="col-md-2 col-sm-12">Map</div>