From a640f7c487eddeb4c460d9c7202fe9fa03f65b09 Mon Sep 17 00:00:00 2001 From: PascalLike <1323093+PascalLike@users.noreply.github.com> Date: Tue, 27 Feb 2024 20:56:02 +0100 Subject: [PATCH] PostGIS MVT provider and docs (#1552) * Adding mvt Postgis provider * Fix test * Update docs * Rename mvt_postgis to mvt_proxy * Update ogcapi-tiles.rst * Update mvt_proxy.py * Remove not implemented method * Fix typo --------- Co-authored-by: Tom Kralidis --- docs/source/data-publishing/ogcapi-tiles.rst | 51 +++- pygeoapi/plugin.py | 1 + pygeoapi/provider/mvt_elastic.py | 8 - pygeoapi/provider/mvt_proxy.py | 256 +++++++++++++++++++ tests/pygeoapi-test-config-apirules.yml | 2 +- 5 files changed, 306 insertions(+), 12 deletions(-) create mode 100644 pygeoapi/provider/mvt_proxy.py diff --git a/docs/source/data-publishing/ogcapi-tiles.rst b/docs/source/data-publishing/ogcapi-tiles.rst index c3a40b6..acd0768 100644 --- a/docs/source/data-publishing/ogcapi-tiles.rst +++ b/docs/source/data-publishing/ogcapi-tiles.rst @@ -20,6 +20,7 @@ pygeoapi core tile providers are listed below, along with supported features. `MVT-tippecanoe`_,❌,✅ `MVT-elastic`_,✅,❌ + `MVT-proxy`_, N/A , N/A Below are specific connection examples based on supported providers. @@ -50,7 +51,7 @@ This code block shows how to configure pygeoapi to read Mapbox vector tiles gene min: 0 max: 5 schemes: - - WorldCRS84Quad + - WebMercatorQuad format: name: pbf mimetype: application/vnd.mapbox-vector-tile @@ -59,7 +60,7 @@ This code block shows how to configure pygeoapi to read Mapbox vector tiles gene On `this tutorial `_ you can find detailed instructions on how-to generate tiles using tippecanoe and integrate them into pygeoapi. MVT-elastic -^^^^^^^^^^^^ +^^^^^^^^^^^ This provider gives support to serving tiles generated using `Elasticsearch `_. These tiles are rendered on-the-fly using the `Elasticsearch Vector tile search API `_. @@ -80,7 +81,7 @@ This code block shows how to configure pygeoapi to read Mapbox vector tiles from min: 0 max: 5 schemes: - - WorldCRS84Quad + - WebMercatorQuad format: name: pbf mimetype: application/vnd.mapbox-vector-tile @@ -88,6 +89,50 @@ This code block shows how to configure pygeoapi to read Mapbox vector tiles from .. tip:: On `this tutorial `_ you can find detailed instructions on publish tiles stored in an Elasticsearch endpoint. +MVT-proxy +^^^^^^^^^ + +This provider gives support to serving tiles from a generic tiles provider `{z}/{x}/{y}`. + +For example, you can get and publish tiles from PostGIS providers like `pg_tileserver `_ +or `martin `_. Both of them render tiles on the fly and provide properties. + +Following block shows how to configure pygeoapi to read Mapbox vector tiles from pg_tileserver endpoint. + +.. code-block:: yaml + + providers: + - type: tile + name: MVT-proxy + data: http://localhost:7800/public.ne_50m_admin_0_countries/{z}/{x}/{y}.mvt + options: + zoom: + min: 0 + max: 15 + schemes: + - WebMercatorQuad + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + +Following code block shows how to configure pygeoapi to read Mapbox vector tiles from martin endpoint. + +.. code-block:: yaml + + providers: + - type: tile + name: MVT-proxy + data: http://localhost:3000/ne_50m_admin_0_countries/{z}/{x}/{y} + options: + zoom: + min: 0 + max: 15 + schemes: + - WebMercatorQuad + format: + name: pbf + mimetype: application/vnd.mapbox-vector-tile + Data access examples -------------------- diff --git a/pygeoapi/plugin.py b/pygeoapi/plugin.py index 312017a..d882a98 100644 --- a/pygeoapi/plugin.py +++ b/pygeoapi/plugin.py @@ -52,6 +52,7 @@ PLUGINS = { 'MongoDB': 'pygeoapi.provider.mongo.MongoProvider', 'MVT-tippecanoe': 'pygeoapi.provider.mvt_tippecanoe.MVTTippecanoeProvider', # noqa: E501 'MVT-elastic': 'pygeoapi.provider.mvt_elastic.MVTElasticProvider', # noqa: E501 + 'MVT-proxy': 'pygeoapi.provider.mvt_proxy.MVTProxyProvider', # noqa: E501 'OracleDB': 'pygeoapi.provider.oracle.OracleProvider', 'OGR': 'pygeoapi.provider.ogr.OGRProvider', 'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider', diff --git a/pygeoapi/provider/mvt_elastic.py b/pygeoapi/provider/mvt_elastic.py index df739c1..fb020c4 100644 --- a/pygeoapi/provider/mvt_elastic.py +++ b/pygeoapi/provider/mvt_elastic.py @@ -249,11 +249,3 @@ class MVTElasticProvider(BaseMVTProvider): content.links = links return content.dict(exclude_none=True) - - def get_vendor_metadata(self, dataset, server_url, layer, tileset, - title, description, keywords, **kwargs): - """ - Gets tile metadata in tilejson format - """ - LOGGER.debug("Get tilejson metadata") - return "" diff --git a/pygeoapi/provider/mvt_proxy.py b/pygeoapi/provider/mvt_proxy.py new file mode 100644 index 0000000..fd80a99 --- /dev/null +++ b/pygeoapi/provider/mvt_proxy.py @@ -0,0 +1,256 @@ +# ================================================================= +# +# Authors: Antonio Cerciello +# +# Copyright (c) 2024 Antonio Cerciello +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging +import requests +from urllib.parse import urlparse + +from pygeoapi.provider.base_mvt import BaseMVTProvider +from pygeoapi.provider.base import (ProviderConnectionError, + ProviderGenericError, + ProviderInvalidQueryError) +from pygeoapi.models.provider.base import ( + TileSetMetadata, LinkType) +from pygeoapi.util import is_url, url_join + +LOGGER = logging.getLogger(__name__) + + +class MVTProxyProvider(BaseMVTProvider): + """ + MVT Proxy Provider + Provider for serving tiles rendered with an external + tiles provider + """ + + def __init__(self, BaseMVTProvider): + """ + Initialize object + + :param provider_def: provider definition + + :returns: pygeoapi.provider.mvt_proxy.pygeoapi/provider/mvt_proxy.py # noqa + """ + + super().__init__(BaseMVTProvider) + + if not is_url(self.data): + msg = 'Wrong input format for MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + url = urlparse(self.data) + baseurl = f'{url.scheme}://{url.netloc}' + param_type = '?f=mvt' + layer = f'/{self.get_layer()}' + + LOGGER.debug('Extracting layer name from URL') + LOGGER.debug(f'Layer: {layer}') + + tilepath = f'{layer}/tiles' + servicepath = f'{tilepath}/{{tileMatrixSetId}}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}{param_type}' # noqa + + self._service_url = url_join(baseurl, servicepath) + + self._service_metadata_url = url_join( + self.service_url.split('{tileMatrix}/{tileRow}/{tileCol}')[0], + 'metadata') + + def __repr__(self): + return f' {self.data}' + + @property + def service_url(self): + return self._service_url + + @property + def service_metadata_url(self): + return self._service_metadata_url + + def get_layer(self): + """ + Extracts layer name from url + + :returns: layer name + """ + + if not is_url(self.data): + msg = 'Wrong input format for MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + url = urlparse(self.data) + + if ('/{z}/{x}/{y}' not in url.path): + msg = 'Wrong input format for MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + layer = url.path.split('/{z}/{x}/{y}')[0] + + LOGGER.debug(layer) + LOGGER.debug('Removing leading "/"') + return layer[1:] + + def get_tiles_service(self, baseurl=None, servicepath=None, + dirpath=None, tile_type=None): + """ + Gets mvt service description + + :param baseurl: base URL of endpoint + :param servicepath: base path of URL + :param dirpath: directory basepath (equivalent of URL) + :param tile_type: tile format type + + :returns: `dict` of item tile service + """ + + super().get_tiles_service(baseurl, servicepath, + dirpath, tile_type) + + self._service_url = servicepath + return self.get_tms_links() + + def get_tiles(self, layer=None, tileset=None, + z=None, y=None, x=None, format_=None): + """ + Gets tile + + :param layer: mvt tile layer + :param tileset: mvt tileset + :param z: z index + :param y: y index + :param x: x index + :param format_: tile format + + :returns: an encoded mvt tile + """ + if format_ == 'mvt': + format_ = self.format_type + + if is_url(self.data): + url = urlparse(self.data) + base_url = f'{url.scheme}://{url.netloc}' + + if url.query: + url_query = f'?{url.query}' + else: + url_query = '' + + resp = None + + try: + with requests.Session() as session: + session.get(base_url) + if '.' in url.path: + resp = session.get(f'{base_url}/{layer}/{z}/{y}/{x}.{format_}{url_query}') # noqa + else: + resp = session.get(f'{base_url}/{layer}/{z}/{y}/{x}{url_query}') # noqa + + resp.raise_for_status() + return resp.content + except requests.exceptions.RequestException as e: + LOGGER.debug(e) + if resp and resp.status_code < 500: + raise ProviderInvalidQueryError # Client is sending an invalid request # noqa + raise ProviderGenericError # Server error + else: + msg = 'Wrong input format for MVT' + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + def get_html_metadata(self, dataset, server_url, layer, tileset, + title, description, keywords, **kwargs): + + service_url = url_join( + server_url, + f'collections/{dataset}/tiles/{tileset}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f=mvt') # noqa + metadata_url = url_join( + server_url, + f'collections/{dataset}/tiles/{tileset}/metadata') + + metadata = dict() + metadata['id'] = dataset + metadata['title'] = title + metadata['tileset'] = tileset + metadata['collections_path'] = service_url + metadata['json_url'] = f'{metadata_url}?f=json' + + return metadata + + def get_default_metadata(self, dataset, server_url, layer, tileset, + title, description, keywords, **kwargs): + + service_url = url_join( + server_url, + f'collections/{dataset}/tiles/{tileset}/{{tileMatrix}}/{{tileRow}}/{{tileCol}}?f=mvt') # noqa + + content = {} + tiling_schemes = self.get_tiling_schemes() + # Default values + tileMatrixSetURI = tiling_schemes[0].tileMatrixSetURI + crs = tiling_schemes[0].crs + # Checking the selected matrix in configured tiling_schemes + for schema in tiling_schemes: + if (schema.tileMatrixSet == tileset): + crs = schema.crs + tileMatrixSetURI = schema.tileMatrixSetURI + + tiling_scheme_url = url_join( + server_url, f'/TileMatrixSets/{schema.tileMatrixSet}') + tiling_scheme_url_type = "application/json" + tiling_scheme_url_title = f'{schema.tileMatrixSet} tile matrix set definition' # noqa + + tiling_scheme = LinkType(href=tiling_scheme_url, + rel="http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", # noqa + type=tiling_scheme_url_type, + title=tiling_scheme_url_title) + + if tiling_scheme is None: + msg = f'Could not identify a valid tiling schema' # noqa + LOGGER.error(msg) + raise ProviderConnectionError(msg) + + content = TileSetMetadata(title=title, description=description, + keywords=keywords, crs=crs, + tileMatrixSetURI=tileMatrixSetURI) + + links = [] + service_url_link_type = "application/vnd.mapbox-vector-tile" + service_url_link_title = f'{tileset} vector tiles for {layer}' + service_url_link = LinkType(href=service_url, rel="item", + type=service_url_link_type, + title=service_url_link_title) + + links.append(tiling_scheme) + links.append(service_url_link) + + content.links = links + + return content.dict(exclude_none=True) diff --git a/tests/pygeoapi-test-config-apirules.yml b/tests/pygeoapi-test-config-apirules.yml index e8752f8..598903d 100644 --- a/tests/pygeoapi-test-config-apirules.yml +++ b/tests/pygeoapi-test-config-apirules.yml @@ -212,7 +212,7 @@ resources: data: tests/data/ne_110m_lakes.geojson id_field: id - type: tile - name: MVT + name: MVT-tippecanoe # data: http://localhost:9000/ne_110m_lakes/{z}/{x}/{y} data: tests/data/tiles/ne_110m_lakes options: