From 3debd0c750e3f0b41f57298f83f2b92c2e046bba Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Sun, 26 Apr 2020 09:57:07 -0400 Subject: [PATCH] update configuration to be resource specific (#393) --- .github/ISSUE_TEMPLATE/bug_report.md | 4 +- CONTRIBUTING.md | 4 +- README.md | 8 +- aws-lambda/README.md | 2 +- aws-lambda/pygeoapi-serverless-config.yml | 11 +- debian/changelog | 2 +- debian/copyright | 8 +- docker/README.md | 20 ++-- docker/default.config.yml | 14 ++- docker/examples/README.md | 2 +- docker/examples/elastic/ES/Dockerfile | 4 +- .../examples/elastic/ES/docker-entrypoint.sh | 2 +- docker/examples/elastic/README.md | 12 +- docker/examples/elastic/docker-compose.yml | 4 +- .../elastic/pygeoapi/docker.config.yml | 12 +- .../elastic/pygeoapi/es-entrypoint.sh | 2 +- docker/examples/simple/my.config.yml | 11 +- docs/source/api-documentation.rst | 2 +- docs/source/configuration.rst | 47 ++++---- .../data-publishing/ogcapi-features.rst | 6 +- docs/source/data-publishing/stac.rst | 13 ++- docs/source/installation.rst | 2 +- docs/source/openapi.rst | 2 +- docs/source/plugins.rst | 2 +- docs/source/running-with-docker.rst | 4 +- docs/source/running.rst | 8 +- docs/source/tour.rst | 2 +- pygeoapi-config.yml | 7 +- pygeoapi/api.py | 110 ++++++++++-------- pygeoapi/flask_app.py | 14 +-- pygeoapi/linked_data.py | 2 +- pygeoapi/openapi.py | 24 ++-- pygeoapi/provider/ogr.py | 6 +- pygeoapi/starlette_app.py | 26 ++--- pygeoapi/static/css/default.css | 2 +- pygeoapi/templates/collection.html | 2 +- pygeoapi/templates/collections.html | 2 +- pygeoapi/templates/item.html | 4 +- pygeoapi/templates/items.html | 2 +- pygeoapi/templates/process.html | 2 +- pygeoapi/templates/processes.html | 2 +- pygeoapi/templates/queryables.html | 2 +- pygeoapi/templates/stac/item.html | 2 +- pygeoapi/templates/stac/root.html | 2 +- pygeoapi/util.py | 17 +++ tests/cite/ogcapi-features/cite.config.yml | 3 +- tests/data/README.md | 4 +- tests/data/hotosm_bdi_waterways.sql | 12 +- tests/pygeoapi-test-config-envvars.yml | 5 +- tests/pygeoapi-test-config.yml | 5 +- tests/pygeoapi-test-ogr-config.yml | 11 +- tests/test_api.py | 12 +- 52 files changed, 275 insertions(+), 215 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 5a5616f..a59b511 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -21,8 +21,8 @@ If applicable, add screenshots to help explain your problem. **Environment** - OS: - - Python version: - - pygeoapi version: + - Python version: + - pygeoapi version: **Additional context** Add any other context about the problem here. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bbfe81d..d62a4a3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,7 +84,7 @@ Your contribution will be under our [license](https://github.com/geopython/pygeo branch, __base your branch off the ``master`` one__. * Note that depending on how long it takes for the dev team to merge your - patch, the copy of ``master`` you worked off of may get out of date! + patch, the copy of ``master`` you worked off of may get out of date! * If you find yourself 'bumping' a pull request that's been sidelined for a while, __make sure you rebase or merge to latest ``master``__ to ensure a speedier resolution. @@ -103,7 +103,7 @@ Your contribution will be under our [license](https://github.com/geopython/pygeo ### Code Formatting -* __Please follow the coding conventions and style used in the pygeoapi repository.__ +* __Please follow the coding conventions and style used in the pygeoapi repository.__ * pygeoapi follows the [PEP-8](http://www.python.org/dev/peps/pep-0008/) guidelines * 80 characters * spaces, not tabs diff --git a/README.md b/README.md index 70d3157..d42ca99 100644 --- a/README.md +++ b/README.md @@ -35,15 +35,15 @@ Try the swagger ui at `http://localhost:5000/openapi` or ```bash -# feature collection metadata +# collection metadata curl http://localhost:5000/ # conformance curl http://localhost:5000/conformance -# feature collection +# collection curl http://localhost:5000/collections/countries -# feature collection limit 100 +# collection limit 100 curl http://localhost:5000/collections/countries/items?limit=100 -# feature +# collection item curl http://localhost:5000/collections/countries/items/1 # number of hits curl http://localhost:5000/collections/countries/items?resulttype=hits diff --git a/aws-lambda/README.md b/aws-lambda/README.md index 532060f..147c568 100644 --- a/aws-lambda/README.md +++ b/aws-lambda/README.md @@ -35,7 +35,7 @@ zappa undeploy -s zappa_settings.json ## node/serverless -The included `serverless.yml` and `pygeoapi-serverless-config.yml` can be used to deploy pygeoapi +The included `serverless.yml` and `pygeoapi-serverless-config.yml` can be used to deploy pygeoapi on AWS Lambda Serverless Environment. This requires Amazon Credentials and the Serverless deployment tool. diff --git a/aws-lambda/pygeoapi-serverless-config.yml b/aws-lambda/pygeoapi-serverless-config.yml index 05638b4..f762402 100644 --- a/aws-lambda/pygeoapi-serverless-config.yml +++ b/aws-lambda/pygeoapi-serverless-config.yml @@ -81,8 +81,9 @@ metadata: instructions: During hours of service. Off on weekends. role: pointOfContact -datasets: +resources: obs: + type: collection title: Observations description: My cool observations keywords: @@ -116,6 +117,7 @@ datasets: y_field: lat ne_110m_populated_places_simple: + type: collection title: Populated Places description: Point symbols with name attributes. Includes all admin-0 capitals and some other major cities. We favor regional significance over population census in determining our selection of places. Use the scale rankings to filter the number of towns that appear on your map. keywords: @@ -143,6 +145,7 @@ datasets: id_field: geonameid lakes: + type: collection title: Large Lakes description: lakes of the world, public domain keywords: @@ -167,6 +170,7 @@ datasets: id_field: id countries: + type: collection title: Countries in the world description: Countries of the world keywords: @@ -191,6 +195,7 @@ datasets: id_field: ogc_fid table: ne_110m_admin_0_countries poi: + type: collection title: Portuguese point of interrest description: Portuguese points of interrest obtained from OpenStreetMap. Dataset includes Madeira and Azores islands keywords: @@ -222,6 +227,7 @@ datasets: table: poi_portugal hotosm_bdi_waterways: + type: collection title: Waterways of Burundi description: Waterways of Burundi, Africa. Dataset timestamp 1st Sep 2018 - Humanitarian OpenStreetMap Team (HOT) keywords: @@ -256,6 +262,7 @@ datasets: table: hotosm_bdi_waterways dutch_georef_stations: + type: collection title: Dutch Georef Stations via OGR WFS description: Locations of RD/GNSS-reference stations from Dutch Kadaster PDOK a.k.a RDInfo. Uses MapServer WFS v2 backend via OGRProvider. keywords: @@ -301,7 +308,7 @@ datasets: id_field: gml_id layer: rdinfo:stations -processes: hello-world: + type: process processor: name: HelloWorld diff --git a/debian/changelog b/debian/changelog index cfa78e4..41faf82 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ pygeoapi (0.7.0-1~bionic0) bionic; urgency=medium - + * Numerous CITE compliance fixes * Elasticsearch: update provider to support ES7 * MongoDB provider implementation diff --git a/debian/copyright b/debian/copyright index b610efb..3671ecd 100644 --- a/debian/copyright +++ b/debian/copyright @@ -5,11 +5,11 @@ Files: * Copyright: Copyright 2019 Tom Kralidis License: MIT Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom + sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: . @@ -21,6 +21,6 @@ License: MIT OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/docker/README.md b/docker/README.md index 2b28e0c..af809e6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,9 +2,9 @@ Docker Image `geopython/pygeoapi:latest` and versions are [available from DockerHub](https://hub.docker.com/r/geopython/pygeoapi). -Each Docker Image contains a default configuration [default.config.yml](default.config.yml) with the project's test data and WFS3/OGC API Features datasets. +Each Docker Image contains a default configuration [default.config.yml](default.config.yml) with the project's test data and OGC API dataset collections. -You can override this default config via Docker Volume mapping or by extending the Docker Image and copying in your config. See an [example for the geoapi demo server](https://github.com/geopython/demo.pygeoapi.io/tree/master/services/pygeoapi) for the latter method. +You can override this default config via Docker Volume mapping or by extending the Docker Image and copying in your config. See an [example for the geoapi demo server](https://github.com/geopython/demo.pygeoapi.io/tree/master/services/pygeoapi) for the latter method. https://github.com/geopython/demo.pygeoapi.io/tree/master/services Depending on your config you may need specific backends to be available. @@ -19,9 +19,9 @@ So the chain is: ``` -There are a number of examples at [several examples](https://github.com/geopython/pygeoapi/blob/master/docker/examples). +There are a number of examples at [several examples](https://github.com/geopython/pygeoapi/blob/master/docker/examples). -### Installation +### Installation Install Docker (Ubuntu) @@ -51,11 +51,11 @@ Run/Create Container $ sudo docker run --name geoapi -p 5000:80 -v $(pwd)/my.config.yml:/pygeoapi/local.config.yml -it geopython/pygeoapi ``` -Go to http://localhost:5000/ and should be up and running. +Go to http://localhost:5000/ and should be up and running. ## Running - Basics -By default this Image will start a `pygeoapi` Docker Container +By default this Image will start a `pygeoapi` Docker Container using `gunicorn` on internal port 80. To run with default built-in config and data: @@ -64,7 +64,7 @@ To run with default built-in config and data: docker run -p 5000:80 -it geopython/pygeoapi run # or simply docker run -p 5000:80 -it geopython/pygeoapi - + # then browse to http://localhost:5000/ ``` @@ -111,13 +111,13 @@ COPY ./my.config.yml /pygeoapi/local.config.yml See how the demo server is setup this way at https://github.com/geopython/demo.pygeoapi.io/tree/master/services/pygeoapi_master - + ## Running - Running on a sub-path By default the `pygeoapi` Docker Image will run from the `root` path `/`. If you need to run from a sub-path and have all internal URLs correct you need to set `SCRIPT_NAME` environment variable. - + For example to run with `my.config.yml` on http://localhost:5000/mypygeoapi: ``` @@ -140,7 +140,7 @@ services: ports: - "5000:80" - + environment: - SCRIPT_NAME=/pygeoapi diff --git a/docker/default.config.yml b/docker/default.config.yml index 5a05a08..ca2d429 100644 --- a/docker/default.config.yml +++ b/docker/default.config.yml @@ -87,8 +87,9 @@ metadata: instructions: During hours of service. Off on weekends. role: pointOfContact -datasets: +resources: obs: + type: collection title: Observations description: My cool observations keywords: @@ -126,6 +127,7 @@ datasets: y_field: lat lakes: + type: collection title: Large Lakes description: lakes of the world, public domain keywords: @@ -149,6 +151,7 @@ datasets: id_field: id countries: + type: collection title: Countries in the world (SpatialLite Provider) description: Countries of the world (SpatialLite) keywords: @@ -174,6 +177,7 @@ datasets: table: ne_110m_admin_0_countries dutch_georef_stations: + type: collection title: Dutch Georef Stations via OGR WFS description: Locations of RD/GNSS-reference stations from Dutch Kadaster PDOK a.k.a RDInfo. Uses MapServer WFS v2 backend via OGRProvider. keywords: @@ -220,6 +224,7 @@ datasets: layer: rdinfo:stations utah_city_locations: + type: collection title: Cities in Utah via OGR WFS description: Data from the state of Utah. Standard demo dataset from the deegree WFS server that is used as backend WFS. keywords: @@ -265,6 +270,7 @@ datasets: layer: app:SGID93_LOCATION_UDOTMap_CityLocations unesco_pois_italy: + type: collection title: Unesco POIs in Italy via OGR WFS description: Unesco Points of Interest in Italy. Using GeoSolutions GeoServer WFS demo-server as backend WFS. keywords: @@ -309,6 +315,7 @@ datasets: layer: unesco:Unesco_point ogr_gpkg_poi: + type: collection title: Portuguese Points of Interest via OGR GPKG description: Portuguese Points of Interest obtained from OpenStreetMap. Dataset includes Madeira and Azores islands. Uses GeoPackage backend via OGR provider. keywords: @@ -354,6 +361,7 @@ datasets: layer: poi_portugal ogr_geojson_lakes: + type: collection title: Large Lakes OGR GeoJSON Driver description: lakes of the world, public domain keywords: @@ -392,6 +400,7 @@ datasets: layer: ne_110m_lakes ogr_addresses_sqlite: + type: collection title: Dutch addresses (subset Otterlo). OGR SQLite Driver description: Dutch addresses subset. keywords: @@ -433,6 +442,7 @@ datasets: layer: ogrgeojson ogr_addresses_gpkg: + type: collection title: Dutch addresses (subset Otterlo). OGR GeoPackage Driver description: Dutch addresses subset. keywords: @@ -472,7 +482,7 @@ datasets: id_field: id layer: OGRGeoJSON -processes: hello-world: + type: process processor: name: HelloWorld diff --git a/docker/examples/README.md b/docker/examples/README.md index bc0f6ac..c48a2b5 100644 --- a/docker/examples/README.md +++ b/docker/examples/README.md @@ -5,5 +5,5 @@ This folder contains the sub-folders: - simple - elastic -The [simple](simple) example will run pygeoapi with Docker with your local config. +The [simple](simple) example will run pygeoapi with Docker with your local config. The [elastic](elastic) example demonstrates a docker compose configuration to run pygeoapi with local ElasticSearch backend. diff --git a/docker/examples/elastic/ES/Dockerfile b/docker/examples/elastic/ES/Dockerfile index a3434da..b987c19 100644 --- a/docker/examples/elastic/ES/Dockerfile +++ b/docker/examples/elastic/ES/Dockerfile @@ -61,12 +61,12 @@ CMD ["/usr/share/elasticsearch/bin/elasticsearch"] ENTRYPOINT ["/docker-entrypoint.sh"] -# we need to run this on host +# we need to run this on host #sudo sysctl -w vm.max_map_count=262144 #check indices #http://localhost:9200/_cat/indices?v #check spatial data #http://localhost:9200/ne_110m_populated_places_simple/ #This docker compose was inspired on: -#https://discuss.elastic.co/t/best-practice-for-creating-an-index-when-an-es-docker-container-starts/126651 +#https://discuss.elastic.co/t/best-practice-for-creating-an-index-when-an-es-docker-container-starts/126651 #docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" es:latest diff --git a/docker/examples/elastic/ES/docker-entrypoint.sh b/docker/examples/elastic/ES/docker-entrypoint.sh index 355025a..8f586a6 100755 --- a/docker/examples/elastic/ES/docker-entrypoint.sh +++ b/docker/examples/elastic/ES/docker-entrypoint.sh @@ -33,4 +33,4 @@ # wait for Elasticsearch to start, then run the setup script to # create and configure the index. exec /usr/share/elasticsearch/bin/wait-for-it.sh localhost:9200 -- /add_data.sh & -exec $@ +exec $@ diff --git a/docker/examples/elastic/README.md b/docker/examples/elastic/README.md index 130ae5a..380c20d 100644 --- a/docker/examples/elastic/README.md +++ b/docker/examples/elastic/README.md @@ -1,7 +1,7 @@ # pygeoapi with ElasticSearch (ES) -These folders contain a Docker Compose configuration necessary to setup a minimal -`pygeoapi` server that uses a local ES backend service. +These folders contain a Docker Compose configuration necessary to setup a minimal +`pygeoapi` server that uses a local ES backend service. This config is only for local development and testing. @@ -10,11 +10,11 @@ This config is only for local development and testing. - official ElasticSearch: **5.6.8** on **CentosOS 7** - ports **9300** and **9200** -ES requires the host system to have its virtual memory +ES requires the host system to have its virtual memory parameter (**max_map_count**) [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html) set as follows: -``` +``` sudo sysctl -w vm.max_map_count=262144 ``` @@ -32,5 +32,5 @@ To build and run the [Docker compose file](docker-compose.yml) in localhost: ``` sudo sysctl -w vm.max_map_count=262144 docker-compose build -docker-compose up -``` +docker-compose up +``` diff --git a/docker/examples/elastic/docker-compose.yml b/docker/examples/elastic/docker-compose.yml index d294e96..c1dec87 100644 --- a/docker/examples/elastic/docker-compose.yml +++ b/docker/examples/elastic/docker-compose.yml @@ -58,7 +58,7 @@ services: build: ./ES container_name: elastic - + ports: - 9300:9300 - 9200:9200 @@ -67,6 +67,6 @@ services: volumes: elastic_search_data: {} - + #NOTE: Host requires changes in configuration to run ES #sudo sysctl -w vm.max_map_count=262144 diff --git a/docker/examples/elastic/pygeoapi/docker.config.yml b/docker/examples/elastic/pygeoapi/docker.config.yml index dbadae1..dec0261 100644 --- a/docker/examples/elastic/pygeoapi/docker.config.yml +++ b/docker/examples/elastic/pygeoapi/docker.config.yml @@ -84,8 +84,9 @@ metadata: instructions: During hours of service. Off on weekends. role: pointOfContact -datasets: +resources: obs: + type: collection title: Observations description: My cool observations keywords: @@ -119,6 +120,7 @@ datasets: y_field: long ne_110m_populated_places_simple: + type: collection title: Populated Places description: Point symbols with name attributes. Includes all admin-0 capitals and some other major cities. We favor regional significance over population census in determining our selection of places. Use the scale rankings to filter the number of towns that appear on your map. keywords: @@ -147,6 +149,7 @@ datasets: id_field: geonameid lakes: + type: collection title: Large Lakes description: lakes of the world, public domain keywords: @@ -169,8 +172,9 @@ datasets: name: GeoJSON data: tests/data/ne_110m_lakes.geojson id_field: id - + countries: + type: collection title: Countries in the world description: Countries of the world keywords: @@ -189,13 +193,13 @@ datasets: temporal: begin: end: null # or empty - provider: + provider: name: SQLiteGPKG data: tests/data/ne_110m_admin_0_countries.sqlite id_field: ogc_fid table: ne_110m_admin_0_countries -processes: hello-world: + type: process processor: name: HelloWorld diff --git a/docker/examples/elastic/pygeoapi/es-entrypoint.sh b/docker/examples/elastic/pygeoapi/es-entrypoint.sh index e81c6a4..1e8f371 100755 --- a/docker/examples/elastic/pygeoapi/es-entrypoint.sh +++ b/docker/examples/elastic/pygeoapi/es-entrypoint.sh @@ -31,7 +31,7 @@ # ================================================================= set +e - + echo "Waiting for ElasticSearch container..." # First wait for ES to be up and then execute the original pygeoapi entrypoint. diff --git a/docker/examples/simple/my.config.yml b/docker/examples/simple/my.config.yml index 6ccdcd1..7c3ed67 100644 --- a/docker/examples/simple/my.config.yml +++ b/docker/examples/simple/my.config.yml @@ -82,8 +82,9 @@ metadata: instructions: During hours of service. Off on weekends. role: pointOfContact -datasets: +resources: obs: + type: collection title: Observations description: My cool observations keywords: @@ -117,6 +118,7 @@ datasets: y_field: long lakes: + type: collection title: Large Lakes description: lakes of the world, public domain keywords: @@ -139,8 +141,9 @@ datasets: name: GeoJSON data: tests/data/ne_110m_lakes.geojson id_field: id - + countries: + type: collection title: Countries in the world description: Countries of the world keywords: @@ -159,13 +162,13 @@ datasets: temporal: begin: end: null # or empty - provider: + provider: name: SQLiteGPKG data: tests/data/ne_110m_admin_0_countries.sqlite id_field: ogc_fid table: ne_110m_admin_0_countries -processes: hello-world: + type: process processor: name: HelloWorld diff --git a/docs/source/api-documentation.rst b/docs/source/api-documentation.rst index bc7a115..d4cc2bf 100644 --- a/docs/source/api-documentation.rst +++ b/docs/source/api-documentation.rst @@ -127,7 +127,7 @@ Hello world example process :members: :private-members: :special-members: - + .. _data Provider: Provider diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index b784e8b..2ab7486 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -16,8 +16,7 @@ pygeoapi configuration contains the following core sections: - ``server``: server-wide settings - ``logging``: logging configuration - ``metadata``: server-wide metadata (contact, licensing, etc.) -- ``datasets``: dataset collections offered by server -- ``processes``: processes offered by server +- ``resources``: dataset collections, processes and stac-collections offered by the server .. note:: `Standard YAML mechanisms `_ can be used (anchors, references, etc.) for reuse and compactness. @@ -44,7 +43,7 @@ The ``server`` section provides directives on binding and high level tuning. language: en-US # default server language cors: true # boolean on whether server should support CORS pretty_print: true # whether JSON responses should be pretty-printed - limit: 10 # server limit on number of features to return + limit: 10 # server limit on number of items to return map: # leaflet map setup for HTML pages url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png attribution: 'Wikimedia maps | Map data © OpenStreetMap contributors' @@ -106,15 +105,21 @@ The ``metadata`` section provides settings for overall service metadata and desc instructions: During hours of service. Off on weekends. role: pointOfContact -``datasets`` -^^^^^^^^^^^^ +``resources`` +^^^^^^^^^^^^^ -The ``datasets`` section lists 1 or more dataset collections to be published by the server. +The ``resources`` section lists 1 or more dataset collections to be published by the server. + +The ``resource.type`` property is required. Allowed types are: +- ``collection`` +- ``process`` +- ``stac-collection`` .. code-block:: yaml - datasets: + resources: obs: + type: collection # REQUIRED (collection, process, or stac-collection) title: Observations # title of dataset description: My cool observations # abstract of dataset keywords: # list of related keywords @@ -151,6 +156,12 @@ The ``datasets`` section lists 1 or more dataset collections to be published by - stn_id - value + hello-world: # name of process + type: collection # REQUIRED (collection, process, or stac-collection) + processor: + name: HelloWorld # Python path of process defition + + .. seealso:: `Linked Data`_ for optionally configuring linked data datasets @@ -158,20 +169,6 @@ The ``datasets`` section lists 1 or more dataset collections to be published by :ref:`plugins` for more information on plugins -``processes`` -^^^^^^^^^^^^^ - -.. code-block:: yaml - - processes: - hello-world: # name of process - processor: - name: HelloWorld # Python path of process defition - -.. note:: - See :ref:`plugins` for more information on plugins - - Using environment variables --------------------------- @@ -196,7 +193,7 @@ Linked Data :align: left :alt: JSON-LD support -pygeoapi supports structured metadata about a deployed instance, and is also capable of presenting feature data as +pygeoapi supports structured metadata about a deployed instance, and is also capable of presenting data as structured data. `JSON-LD`_ equivalents are available for each HTML page, and are embedded as data blocks within the corresponding page for search engine optimisation (SEO). Tools such as the `Google Structured Data Testing Tool`_ can be used to check the structured representations. @@ -208,11 +205,11 @@ This metadata is included automatically, and is sufficient for inclusion in majo For collections, at the level of an item or items, by default the JSON-LD representation adds: - The GeoJSON JSON-LD `vocabulary and context `_ to the ``@context``. -- An ``@id`` for each feature in a collection, that is the URL for that feature (resolving to its HTML representation +- An ``@id`` for each item in a collection, that is the URL for that item (resolving to its HTML representation in pygeoapi) .. note:: - While this is enough to provide valid RDF (as GeoJSON-LD), it does not allow the *properties* of your features to be + While this is enough to provide valid RDF (as GeoJSON-LD), it does not allow the *properties* of your items to be unambiguously interpretable. pygeoapi currently allows for the extension of the ``@context`` to allow properties to be aliased to terms from @@ -251,7 +248,7 @@ also expressed as ``_. This example demonstrates how to use this feature with a CSV data provider, using included sample data. The implementation of JSON-LD structured data is available for any data provider but is currently limited to defining a -``@context``. Relationships between features can be expressed but is dependent on such relationships being expressed +``@context``. Relationships between items can be expressed but is dependent on such relationships being expressed by the dataset provider, not pygeoapi. Summary diff --git a/docs/source/data-publishing/ogcapi-features.rst b/docs/source/data-publishing/ogcapi-features.rst index cdd12f2..027af45 100644 --- a/docs/source/data-publishing/ogcapi-features.rst +++ b/docs/source/data-publishing/ogcapi-features.rst @@ -104,7 +104,7 @@ PostgreSQL provider: name: PostgreSQL - data: + data: host: 127.0.0.1 dbname: test user: postgres @@ -145,13 +145,13 @@ GeoPackage file: Data access examples -------------------- -- list all datasets +- list all collections - http://localhost:5000/collections - overview of dataset - http://localhost:5000/collections/foo - browse features - http://localhost:5000/collections/foo/items -- paging +- paging - http://localhost:5000/collections/foo/items?startIndex=10&limit=10 - CSV outputs - http://localhost:5000/collections/foo/items?f=csv diff --git a/docs/source/data-publishing/stac.rst b/docs/source/data-publishing/stac.rst index 2496f3b..031ee99 100644 --- a/docs/source/data-publishing/stac.rst +++ b/docs/source/data-publishing/stac.rst @@ -15,11 +15,14 @@ to the given directory and specifying allowed file types: .. code-block:: yaml - provider: - name: FileSystem - data: /Users/tomkralidis/Dev/data/gdps - file_types: - - .grib2 + my-stac-resource: + type: stac-collection + ... + provider: + name: FileSystem + data: /Users/tomkralidis/Dev/data/gdps + file_types: + - .grib2 .. note:: diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 35c9d3e..0a4b61e 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -61,7 +61,7 @@ Conda .. code-block:: bash - conda install -c conda-forge pygeoapi + conda install -c conda-forge pygeoapi UbuntuGIS --------- diff --git a/docs/source/openapi.rst b/docs/source/openapi.rst index c9287cd..00a5483 100644 --- a/docs/source/openapi.rst +++ b/docs/source/openapi.rst @@ -53,7 +53,7 @@ paging and sorting: .. image:: /_static/openapi_get_item.png -For each feature in our dataset we have a specific identifier. Notice that the identifier is not part of the GeoJSON +For each item in our dataset we have a specific identifier. Notice that the identifier is not part of the GeoJSON properties, but is provided as a GeoJSON root property of ``id``. .. image:: /_static/openapi_get_item_id.png diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 6de236d..d7c3b9a 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -54,7 +54,7 @@ The below template provides a minimal example (let's call the file ``mycooldata. class MyCoolDataProvider(BaseProvider): """My cool data provider""" - + def __init__(self, provider_def): """Inherit from parent class""" diff --git a/docs/source/running-with-docker.rst b/docs/source/running-with-docker.rst index 803ff46..3520915 100644 --- a/docs/source/running-with-docker.rst +++ b/docs/source/running-with-docker.rst @@ -62,7 +62,7 @@ Or you can create a ``Dockerfile`` extending the base image and **copy** in your .. code-block:: dockerfile - FROM geopython/pygeoapi:latest + FROM geopython/pygeoapi:latest COPY ./my.config.yml /pygeoapi/local.config.yml A corresponding example can be found in https://github.com/geopython/demo.pygeoapi.io/tree/master/services/pygeoapi_master @@ -72,7 +72,7 @@ Deploying on a sub-path By default the ``pygeoapi`` Docker image will run from the ``root`` path (``/``). If you need to run from a sub-path and have all internal URLs properly configured, you can set the ``SCRIPT_NAME`` environment variable. - + For example to run with ``my.config.yml`` on ``http://localhost:5000/mypygeoapi``: .. code-block:: bash diff --git a/docs/source/running.rst b/docs/source/running.rst index a413690..8e11ef3 100644 --- a/docs/source/running.rst +++ b/docs/source/running.rst @@ -25,11 +25,11 @@ Flask WSGI Web Server Gateway Interface (WSGI) is a standard for how web servers communicate with Python applications. By having a WSGI server, HTTP requests are processed into threads/processes for better performance. Flask is a WSGI implementation which pygeoapi utilizes to communicate with the core API. - + .. code-block:: bash - + HTTP request <--> Flask (pygeoapi/flask_app.py) <--> pygeoapi API (pygeoapi/api.py) - + The Flask WSGI server can be run as follows: @@ -116,7 +116,7 @@ Gunicorn and Flask Gunicorn and Flask is simple to run: .. code-block:: bash - + gunicorn pygeoapi.flask_app:APP .. note:: diff --git a/docs/source/tour.rst b/docs/source/tour.rst index 4166db2..0db8521 100644 --- a/docs/source/tour.rst +++ b/docs/source/tour.rst @@ -72,7 +72,7 @@ This page displays a map and tabular view of the data. Features are clickable o allowing the user to drill into more information about the feature. The table also allows for drilling into a feature by clicking the link in a given table row. -Let's checkout the feature close to `Toronto, Ontario, Canada`_. +Let's inspect the feature close to `Toronto, Ontario, Canada`_. Collection item diff --git a/pygeoapi-config.yml b/pygeoapi-config.yml index 0335787..507b327 100644 --- a/pygeoapi-config.yml +++ b/pygeoapi-config.yml @@ -80,8 +80,9 @@ metadata: instructions: During hours of service. Off on weekends. role: pointOfContact -datasets: +resources: obs: + type: collection title: Observations description: My cool observations keywords: @@ -119,6 +120,7 @@ datasets: y_field: lat lakes: + type: collection title: Large Lakes description: lakes of the world, public domain keywords: @@ -142,6 +144,7 @@ datasets: id_field: id test-data: + type: stac-collection title: pygeoapi test data description: pygeoapi test data keywords: @@ -165,7 +168,7 @@ datasets: - .csv - .grib2 -processes: hello-world: + type: process processor: name: HelloWorld diff --git a/pygeoapi/api.py b/pygeoapi/api.py index 6f2ed7a..b2cc65c 100644 --- a/pygeoapi/api.py +++ b/pygeoapi/api.py @@ -47,8 +47,8 @@ from pygeoapi.plugin import load_plugin, PLUGINS from pygeoapi.provider.base import ( ProviderGenericError, ProviderConnectionError, ProviderNotFoundError, ProviderQueryError, ProviderItemNotFoundError) -from pygeoapi.util import (dategetter, json_serial, render_j2_template, - str2bool, TEMPLATES) +from pygeoapi.util import (dategetter, filter_dict_by_key_value, json_serial, + render_j2_template, str2bool, TEMPLATES) LOGGER = logging.getLogger(__name__) @@ -266,7 +266,7 @@ class API: @jsonldify def describe_collections(self, headers_, format_, dataset=None): """ - Provide feature collection metadata + Provide collection metadata :param headers_: copy of HEADERS object :param format_: format of requests, @@ -289,21 +289,22 @@ class API: 'links': [] } - if all([dataset is not None, - dataset not in self.config['datasets'].keys()]): + collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'collection') + if all([dataset is not None, dataset not in collections.keys()]): exception = { 'code': 'InvalidParameterValue', - 'description': 'Invalid feature collection' + 'description': 'Invalid collection' } LOGGER.error(exception) return headers_, 400, json.dumps(exception) LOGGER.debug('Creating collections') - for k, v in self.config['datasets'].items(): + for k, v in collections.items(): collection = {'links': []} collection['id'] = k - collection['itemType'] = 'feature' + collection['itemType'] = 'Feature' collection['title'] = v['title'] collection['description'] = v['description'] collection['keywords'] = v['keywords'] @@ -363,21 +364,21 @@ class API: collection['links'].append({ 'type': 'application/geo+json', 'rel': 'items', - 'title': 'Features as GeoJSON', + 'title': 'items as GeoJSON', 'href': '{}/collections/{}/items?f=json'.format( self.config['server']['url'], k) }) collection['links'].append({ 'type': 'application/ld+json', 'rel': 'items', - 'title': 'Features as RDF (GeoJSON-LD)', + 'title': 'items as RDF (GeoJSON-LD)', 'href': '{}/collections/{}/items?f=jsonld'.format( self.config['server']['url'], k) }) collection['links'].append({ 'type': 'text/html', 'rel': 'items', - 'title': 'Features as HTML', + 'title': 'Items as HTML', 'href': '{}/collections/{}/items?f=html'.format( self.config['server']['url'], k) }) @@ -486,7 +487,7 @@ class API: return headers_, 400, json.dumps(exception) if any([dataset is None, - dataset not in self.config['datasets'].keys()]): + dataset not in self.config['resources'].keys()]): exception = { 'code': 'InvalidParameterValue', @@ -499,7 +500,7 @@ class API: LOGGER.debug('Loading provider') try: p = load_plugin('provider', - self.config['datasets'][dataset]['provider']) + self.config['resources'][dataset]['provider']) except ProviderConnectionError: exception = { 'code': 'NoApplicableCode', @@ -534,7 +535,7 @@ class API: }) if format_ == 'html': # render - queryables['title'] = self.config['datasets'][dataset]['title'] + queryables['title'] = self.config['resources'][dataset]['title'] headers_['Content-Type'] = 'text/html' content = render_j2_template(self.config, 'queryables.html', queryables) @@ -545,7 +546,7 @@ class API: def get_collection_items(self, headers, args, dataset, pathinfo=None): """ - Queries feature collection + Queries collection :param headers: dict of HTTP headers :param args: dict of HTTP request parameters @@ -563,10 +564,13 @@ class API: formats = FORMATS formats.extend(f.lower() for f in PLUGINS['formatter'].keys()) - if dataset not in self.config['datasets'].keys(): + collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): exception = { 'code': 'InvalidParameterValue', - 'description': 'Invalid feature collection' + 'description': 'Invalid collection' } LOGGER.error(exception) return headers_, 400, json.dumps(exception, default=json_serial) @@ -665,8 +669,8 @@ class API: datetime_invalid = False if (datetime_ is not None and - 'temporal' in self.config['datasets'][dataset]['extents']): - te = self.config['datasets'][dataset]['extents']['temporal'] + 'temporal' in collections[dataset]['extents']): + te = collections[dataset]['extents']['temporal'] if te['begin'].tzinfo is None: te['begin'] = te['begin'].replace(tzinfo=pytz.UTC) @@ -720,7 +724,7 @@ class API: LOGGER.debug('Loading provider') try: p = load_plugin('provider', - self.config['datasets'][dataset]['provider']) + collections[dataset]['provider']) except ProviderConnectionError: exception = { 'code': 'NoApplicableCode', @@ -869,7 +873,7 @@ class API: content['links'].append( { 'type': 'application/json', - 'title': self.config['datasets'][dataset]['title'], + 'title': collections[dataset]['title'], 'rel': 'collection', 'href': '{}/collections/{}'.format( self.config['server']['url'], dataset) @@ -906,7 +910,7 @@ class API: data=content, options={ 'provider_def': - self.config['datasets'][dataset]['provider'] + collections[dataset]['provider'] } ) @@ -927,13 +931,13 @@ class API: @pre_process def get_collection_item(self, headers_, format_, dataset, identifier): """ - Get a single feature + Get a single collection item :param headers_: copy of HEADERS object :param format_: format of requests, pre checked by pre_process decorator :param dataset: dataset name - :param identifier: feature identifier + :param identifier: item identifier :returns: tuple of headers, status code, content """ @@ -948,17 +952,19 @@ class API: LOGGER.debug('Processing query parameters') - if dataset not in self.config['datasets'].keys(): + collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'collection') + + if dataset not in collections.keys(): exception = { 'code': 'InvalidParameterValue', - 'description': 'Invalid feature collection' + 'description': 'Invalid collection' } LOGGER.error(exception) return headers_, 400, json.dumps(exception) LOGGER.debug('Loading provider') - p = load_plugin('provider', - self.config['datasets'][dataset]['provider']) + p = load_plugin('provider', collections[dataset]['provider']) try: LOGGER.debug('Fetching id {}'.format(identifier)) @@ -1021,7 +1027,7 @@ class API: }, { 'rel': 'collection', 'type': 'application/json', - 'title': self.config['datasets'][dataset]['title'], + 'title': collections[dataset]['title'], 'href': '{}/collections/{}'.format( self.config['server']['url'], dataset) }, { @@ -1040,7 +1046,7 @@ class API: if format_ == 'html': # render headers_['Content-Type'] = 'text/html' - content['title'] = self.config['datasets'][dataset]['title'] + content['title'] = collections[dataset]['title'] content = render_j2_template(self.config, 'item.html', content) return headers_, 200, content @@ -1082,18 +1088,20 @@ class API: 'links': [] } - for key, value in self.config['datasets'].items(): - if value['provider']['name'] == 'FileSystem': - content['links'].append({ - 'rel': 'collection', - 'href': '{}/{}?f=json'.format(stac_url, key), - 'type': 'application/json' - }) - content['links'].append({ - 'rel': 'collection', - 'href': '{}/{}'.format(stac_url, key), - 'type': 'text/html' - }) + stac_collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'stac-collection') + + for key, value in stac_collections.items(): + content['links'].append({ + 'rel': 'collection', + 'href': '{}/{}?f=json'.format(stac_url, key), + 'type': 'application/json' + }) + content['links'].append({ + 'rel': 'collection', + 'href': '{}/{}'.format(stac_url, key), + 'type': 'text/html' + }) if format_ == 'html': # render headers_['Content-Type'] = 'text/html' @@ -1120,7 +1128,10 @@ class API: if dir_tokens: dataset = dir_tokens[0] - if dataset not in self.config['datasets']: + stac_collections = filter_dict_by_key_value(self.config['resources'], + 'type', 'stac-collection') + + if dataset not in stac_collections: exception = { 'code': 'NotFound', 'description': 'collection not found' @@ -1130,8 +1141,7 @@ class API: LOGGER.debug('Loading provider') try: - p = load_plugin('provider', - self.config['datasets'][dataset]['provider']) + p = load_plugin('provider', stac_collections[dataset]['provider']) except ProviderConnectionError as err: LOGGER.error(err) exception = { @@ -1143,7 +1153,7 @@ class API: id_ = '{}-stac'.format(dataset) stac_version = '0.6.2' - description = self.config['datasets'][dataset]['description'] + description = stac_collections[dataset]['description'] content = { 'id': id_, @@ -1174,7 +1184,7 @@ class API: if isinstance(stac_data, dict): content.update(stac_data) - content['links'].extend(self.config['datasets'][dataset]['links']) + content['links'].extend(stac_collections[dataset]['links']) if format_ == 'html': # render headers_['Content-Type'] = 'text/html' @@ -1217,7 +1227,8 @@ class API: LOGGER.error(exception) return headers_, 400, json.dumps(exception) - processes_config = self.config.get('processes', {}) + processes_config = filter_dict_by_key_value(self.config['resources'], + 'type', 'process') if processes_config: if process is not None: @@ -1288,7 +1299,8 @@ class API: LOGGER.error(exception) return headers_, 400, json.dumps(exception) - processes = self.config.get('processes', {}) + processes = filter_dict_by_key_value(self.config['resources'], + 'type', 'process') if process not in processes: exception = { diff --git a/pygeoapi/flask_app.py b/pygeoapi/flask_app.py index a8e5277..a8f3370 100644 --- a/pygeoapi/flask_app.py +++ b/pygeoapi/flask_app.py @@ -189,21 +189,21 @@ def get_collection_queryables(name=None): return response -@APP.route('/collections//items') -@APP.route('/collections//items/') -def dataset(feature_collection, feature=None): +@APP.route('/collections//items') +@APP.route('/collections//items/') +def dataset(collection_id, item_id=None): """ - OGC open api collections/{dataset}/items/{feature} access point + OGC open api collections/{dataset}/items/{item} access point :returns: HTTP response """ - if feature is None: + if item_id is None: headers, status_code, content = api_.get_collection_items( - request.headers, request.args, feature_collection) + request.headers, request.args, collection_id) else: headers, status_code, content = api_.get_collection_item( - request.headers, request.args, feature_collection, feature) + request.headers, request.args, collection_id, item_id) response = make_response(content, status_code) diff --git a/pygeoapi/linked_data.py b/pygeoapi/linked_data.py index 4d3f124..14fb2f8 100644 --- a/pygeoapi/linked_data.py +++ b/pygeoapi/linked_data.py @@ -174,7 +174,7 @@ def geojson2geojsonld(config, data, dataset, identifier=None): :returns: string of rendered JSON (GeoJSON-LD) """ - context = config['datasets'][dataset].get('context', []) + context = config['resources'][dataset].get('context', []) data['id'] = ( '{}/collections/{}/items/{}' if identifier else '{}/collections/{}/items' diff --git a/pygeoapi/openapi.py b/pygeoapi/openapi.py index e05efef..4d86450 100644 --- a/pygeoapi/openapi.py +++ b/pygeoapi/openapi.py @@ -35,7 +35,7 @@ import click import yaml from pygeoapi.plugin import load_plugin -from pygeoapi.util import yaml_load +from pygeoapi.util import filter_dict_by_key_value, yaml_load LOGGER = logging.getLogger(__name__) @@ -198,8 +198,8 @@ def get_oas_30(cfg): paths['/collections'] = { 'get': { - 'summary': 'Feature Collections', - 'description': 'Feature Collections', + 'summary': 'Collections', + 'description': 'Collections', 'tags': ['server'], 'parameters': [ {'$ref': '#/components/parameters/f'} @@ -276,10 +276,10 @@ def get_oas_30(cfg): items_f['schema']['enum'].append('csv') LOGGER.debug('setting up datasets') - for k, v in cfg['datasets'].items(): - if v['provider']['name'] == 'FileSystem': - continue + collections = filter_dict_by_key_value(cfg['resources'], + 'type', 'collection') + for k, v in collections.items(): collection_name_path = '/collections/{}'.format(k) tag = { 'name': k, @@ -298,7 +298,7 @@ def get_oas_30(cfg): paths[collection_name_path] = { 'get': { - 'summary': 'Get feature collection metadata'.format(v['title']), # noqa + 'summary': 'Get collection metadata'.format(v['title']), # noqa 'description': v['description'], 'tags': [k], 'parameters': [ @@ -317,7 +317,7 @@ def get_oas_30(cfg): paths[items_path] = { 'get': { - 'summary': 'Get {} features'.format(v['title']), + 'summary': 'Get {} items'.format(v['title']), 'description': v['description'], 'tags': [k], 'parameters': [ @@ -336,7 +336,7 @@ def get_oas_30(cfg): } } - p = load_plugin('provider', cfg['datasets'][k]['provider']) + p = load_plugin('provider', collections[k]['provider']) if p.fields: queryables_path = '{}/queryables'.format(collection_name_path) @@ -398,9 +398,9 @@ def get_oas_30(cfg): 'explode': False }) - paths['{}/items/{{featureId}}'.format(collection_name_path)] = { + paths['{}/items/{{itemId}}'.format(collection_name_path)] = { 'get': { - 'summary': 'Get {} feature by id'.format(v['title']), + 'summary': 'Get {} item by id'.format(v['title']), 'description': v['description'], 'tags': [k], 'parameters': [ @@ -446,7 +446,7 @@ def get_oas_30(cfg): } } - processes = cfg.get('processes', {}) + processes = filter_dict_by_key_value(cfg['resources'], 'type', 'process') if processes: for k, v in processes.items(): diff --git a/pygeoapi/provider/ogr.py b/pygeoapi/provider/ogr.py index 32241e0..e591e2e 100644 --- a/pygeoapi/provider/ogr.py +++ b/pygeoapi/provider/ogr.py @@ -145,8 +145,8 @@ class OGRProvider(BaseProvider): 'EPSG:4326').split(':')[1]) # Optional coordinate transformation inward (requests) and - # outward (responses) when the source layers and WFS3 collections - # differ in EPSG-codes. + # outward (responses) when the source layers and + # OGC API - Features collections differ in EPSG-codes. self.transform_in = None self.transform_out = None if self.source_srs != self.target_srs: @@ -220,7 +220,7 @@ class OGRProvider(BaseProvider): {}'.format(source_type) LOGGER.error(msg) # ignore errors for ESRIJSON not having geometry member - # see https://github.com/OSGeo/gdal/commit/38b0feed67f80ded32be6c508323d862e1a14474 # noqa + # see https://github.com/OSGeo/gdal/commit/38b0feed67f80ded32be6c508323d862e1a14474 # noqa self.conn = _ignore_gdal_error( self.driver, 'Open', self.data_def['source'], 0) if not self.conn: diff --git a/pygeoapi/starlette_app.py b/pygeoapi/starlette_app.py index 47c73b6..62d13c7 100644 --- a/pygeoapi/starlette_app.py +++ b/pygeoapi/starlette_app.py @@ -174,28 +174,28 @@ async def get_collection_queryables(request: Request, name=None): return response -@app.route('/collections/{feature_collection}/items') -@app.route('/collections/{feature_collection}/items/') -@app.route('/collections/{feature_collection}/items/{feature}') -@app.route('/collections/{feature_collection}/items/{feature}/') -async def dataset(request: Request, feature_collection=None, feature=None): +@app.route('/collections/{collection_id}/items') +@app.route('/collections/{collection_id}/items/') +@app.route('/collections/{collection_id}/items/{item_id}') +@app.route('/collections/{collection_id}/items/{item_id}/') +async def dataset(request: Request, collection_id=None, item_id=None): """ - OGC open api collections/{dataset}/items/{feature} access point + OGC open api collections/{dataset}/items/{item_id} access point :returns: Starlette HTTP Response """ - if 'feature_collection' in request.path_params: - feature_collection = request.path_params['feature_collection'] - if 'feature' in request.path_params: - feature = request.path_params['feature'] - if feature is None: + if 'collection_id' in request.path_params: + collection_id = request.path_params['collection_id'] + if 'item_id' in request.path_params: + item_id = request.path_params['item_id'] + if item_id is None: headers, status_code, content = api_.get_collection_items( request.headers, request.query_params, - feature_collection, pathinfo=request.scope['path']) + collection_id, pathinfo=request.scope['path']) else: headers, status_code, content = api_.get_collection_item( - request.headers, request.query_params, feature_collection, feature) + request.headers, request.query_params, collection_id, item_id) response = Response(content=content, status_code=status_code) diff --git a/pygeoapi/static/css/default.css b/pygeoapi/static/css/default.css index 9afc7cc..33fc4fb 100644 --- a/pygeoapi/static/css/default.css +++ b/pygeoapi/static/css/default.css @@ -12,7 +12,7 @@ main { .crumbs { background-color:rgb(230, 230, 230); - padding: 6px; + padding: 6px; } .crumbs a { diff --git a/pygeoapi/templates/collection.html b/pygeoapi/templates/collection.html index 485d788..3cafa71 100644 --- a/pygeoapi/templates/collection.html +++ b/pygeoapi/templates/collection.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {% block title %}{{ super() }} {{ data['title'] }} {% endblock %} -{% block crumbs %}{{ super() }} +{% block crumbs %}{{ super() }} / Collections / {{ data['title'] }} {% endblock %} diff --git a/pygeoapi/templates/collections.html b/pygeoapi/templates/collections.html index 6e67b6f..4cef6df 100644 --- a/pygeoapi/templates/collections.html +++ b/pygeoapi/templates/collections.html @@ -17,7 +17,7 @@ - {% for k, v in config['datasets'].items() %} + {% for k, v in filter_dict_by_key_value(config['resources'], 'type', 'collection').items() %} diff --git a/pygeoapi/templates/item.html b/pygeoapi/templates/item.html index 390cac8..a3f08be 100644 --- a/pygeoapi/templates/item.html +++ b/pygeoapi/templates/item.html @@ -11,13 +11,13 @@ {% endif %} {%- endmacro %} {% block title %}{{ super() }} {{ data['title'] }} - {{ data['id'] }}{% endblock %} -{% block crumbs %}{{ super() }} +{% block crumbs %}{{ super() }} / Collections {% for link in data['links'] %} {% if link.rel == 'collection' %} / {{ link['title'] }} {% endif %} -{% endfor %} +{% endfor %} / Items / Item {{ data['id'] }} {% endblock %} diff --git a/pygeoapi/templates/items.html b/pygeoapi/templates/items.html index 0fc905d..872b49a 100644 --- a/pygeoapi/templates/items.html +++ b/pygeoapi/templates/items.html @@ -20,7 +20,7 @@

{% for l in data['links'] if l.rel == 'collection' %} {{ l['title'] }} {% endfor %}

Items in this collection.

-
+
{% if data['features'] %}
diff --git a/pygeoapi/templates/process.html b/pygeoapi/templates/process.html index 9590583..ae8daf7 100644 --- a/pygeoapi/templates/process.html +++ b/pygeoapi/templates/process.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {% block title %}{{ super() }} {{ data['title'] }} {% endblock %} -{% block crumbs %}{{ super() }} +{% block crumbs %}{{ super() }} / Processes / {{ data['title'] }} {% endblock %} diff --git a/pygeoapi/templates/processes.html b/pygeoapi/templates/processes.html index bedbb56..7497fac 100644 --- a/pygeoapi/templates/processes.html +++ b/pygeoapi/templates/processes.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {% block title %}{{ super() }} Processes {% endblock %} -{% block crumbs %}{{ super() }} +{% block crumbs %}{{ super() }} / Processes {% endblock %} {% block body %} diff --git a/pygeoapi/templates/queryables.html b/pygeoapi/templates/queryables.html index b0ec7d7..f9cc5a2 100644 --- a/pygeoapi/templates/queryables.html +++ b/pygeoapi/templates/queryables.html @@ -1,6 +1,6 @@ {% extends "base.html" %} {% block title %}{{ super() }} {{ data['title'] }} {% endblock %} -{% block crumbs %}{{ super() }} +{% block crumbs %}{{ super() }} / Collections / {{ data['title'] }} / Queryables diff --git a/pygeoapi/templates/stac/item.html b/pygeoapi/templates/stac/item.html index 92f2693..a79b155 100644 --- a/pygeoapi/templates/stac/item.html +++ b/pygeoapi/templates/stac/item.html @@ -26,7 +26,7 @@

Assets

- +
diff --git a/pygeoapi/templates/stac/root.html b/pygeoapi/templates/stac/root.html index 0b2a5e8..b8c04cc 100644 --- a/pygeoapi/templates/stac/root.html +++ b/pygeoapi/templates/stac/root.html @@ -24,7 +24,7 @@ - {% for k, v in config['datasets'].items() %} + {% for k, v in filter_dict_by_key_value(config['resources'], 'type', 'stac-collection').items() %} {% if v['provider']['name'] == 'FileSystem' %}
URL
diff --git a/pygeoapi/util.py b/pygeoapi/util.py index 43d4cf2..5e92305 100644 --- a/pygeoapi/util.py +++ b/pygeoapi/util.py @@ -220,6 +220,9 @@ def render_j2_template(config, template, data): env.filters['get_breadcrumbs'] = get_breadcrumbs env.globals.update(get_breadcrumbs=get_breadcrumbs) + env.filters['filter_dict_by_key_value'] = filter_dict_by_key_value + env.globals.update(filter_dict_by_key_value=filter_dict_by_key_value) + template = env.get_template(template) return template.render(config=config, data=data, version=__version__) @@ -261,3 +264,17 @@ def get_breadcrumbs(urlpath): }) return links + + +def filter_dict_by_key_value(dict_, key, value): + """ + helper generator function to filter a dict by a dict key + + :param dict_: ``dict`` + :param key: dict key + :param value: dict key value + + :returns: filtered ``dict`` + """ + + return {k: v for (k, v) in dict_.items() if v[key] == value} diff --git a/tests/cite/ogcapi-features/cite.config.yml b/tests/cite/ogcapi-features/cite.config.yml index 1605d8a..61d8897 100644 --- a/tests/cite/ogcapi-features/cite.config.yml +++ b/tests/cite/ogcapi-features/cite.config.yml @@ -51,8 +51,9 @@ metadata: instructions: During hours of service. Off on weekends. role: pointOfContact -datasets: +resources: canada-hydat-daily-mean-02hc003: + type: collection title: Daily Mean of Water Level or Flow description: The daily mean is the average of all unit values for a given day. keywords: [Daily, Daily Mean, Water Level, Flow, Discharge] diff --git a/tests/data/README.md b/tests/data/README.md index 76507a0..84949a0 100644 --- a/tests/data/README.md +++ b/tests/data/README.md @@ -28,7 +28,7 @@ This directory provides test data to demonstrate functionality. ### `obs.csv` - source: MapServer msautotest suite -- URL: [https://github.com/mapserver/mapserver/blob/branch-7-0/msautotest/wxs/data/obs.csv](https://github.com/mapserver/mapserver/blob/branch-7-0/msautotest/wxs/data/obs.csv) +- URL: [https://github.com/mapserver/mapserver/blob/branch-7-0/msautotest/wxs/data/obs.csv](https://github.com/mapserver/mapserver/blob/branch-7-0/msautotest/wxs/data/obs.csv) - Copyright (c) 2008-2018 Open Source Geospatial Foundation - Copyright (c) 1996-2008 Regents of the University of Minnesota @@ -44,7 +44,7 @@ This directory provides test data to demonstrate functionality. - source: OpenStreetMap - Humanitarian OpenStreetMap Team (HOT) - URL: [hotosm_bdi_waterways](https://data.humdata.org/dataset/hotosm_bdi_waterways) - Waterways of Burundi -- Date of dataset: Sep 01, 2018 +- Date of dataset: Sep 01, 2018 - Location: Burundi, Africa ### `CMC_glb_*.grib2` diff --git a/tests/data/hotosm_bdi_waterways.sql b/tests/data/hotosm_bdi_waterways.sql index 4c119e0..d2cc9a2 100644 --- a/tests/data/hotosm_bdi_waterways.sql +++ b/tests/data/hotosm_bdi_waterways.sql @@ -42,42 +42,42 @@ COMMENT ON SCHEMA topology IS 'PostGIS Topology schema'; -- --- Name: hstore; Type: EXTENSION; Schema: -; Owner: +-- Name: hstore; Type: EXTENSION; Schema: -; Owner: -- CREATE EXTENSION IF NOT EXISTS hstore WITH SCHEMA public; -- --- Name: EXTENSION hstore; Type: COMMENT; Schema: -; Owner: +-- Name: EXTENSION hstore; Type: COMMENT; Schema: -; Owner: -- COMMENT ON EXTENSION hstore IS 'data type for storing sets of (key, value) pairs'; -- --- Name: postgis; Type: EXTENSION; Schema: -; Owner: +-- Name: postgis; Type: EXTENSION; Schema: -; Owner: -- CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; -- --- Name: EXTENSION postgis; Type: COMMENT; Schema: -; Owner: +-- Name: EXTENSION postgis; Type: COMMENT; Schema: -; Owner: -- COMMENT ON EXTENSION postgis IS 'PostGIS geometry, geography, and raster spatial types and functions'; -- --- Name: postgis_topology; Type: EXTENSION; Schema: -; Owner: +-- Name: postgis_topology; Type: EXTENSION; Schema: -; Owner: -- CREATE EXTENSION IF NOT EXISTS postgis_topology WITH SCHEMA topology; -- --- Name: EXTENSION postgis_topology; Type: COMMENT; Schema: -; Owner: +-- Name: EXTENSION postgis_topology; Type: COMMENT; Schema: -; Owner: -- COMMENT ON EXTENSION postgis_topology IS 'PostGIS topology spatial types and functions'; diff --git a/tests/pygeoapi-test-config-envvars.yml b/tests/pygeoapi-test-config-envvars.yml index 717f94f..c8e7bab 100644 --- a/tests/pygeoapi-test-config-envvars.yml +++ b/tests/pygeoapi-test-config-envvars.yml @@ -80,8 +80,9 @@ metadata: instructions: During hours of service. Off on weekends. role: pointOfContact -datasets: +resources: obs: + type: collection title: Observations description: My cool observations keywords: @@ -114,7 +115,7 @@ datasets: x_field: long y_field: lat -processes: hello-world: + type: process processor: name: HelloWorld diff --git a/tests/pygeoapi-test-config.yml b/tests/pygeoapi-test-config.yml index fd32f46..f9ffd0c 100644 --- a/tests/pygeoapi-test-config.yml +++ b/tests/pygeoapi-test-config.yml @@ -80,8 +80,9 @@ metadata: instructions: During hours of service. Off on weekends. role: pointOfContact -datasets: +resources: obs: + type: collection title: Observations description: My cool observations keywords: @@ -125,7 +126,7 @@ datasets: x_field: long y_field: lat -processes: hello-world: + type: process processor: name: HelloWorld diff --git a/tests/pygeoapi-test-ogr-config.yml b/tests/pygeoapi-test-ogr-config.yml index abd0d6c..b91feb3 100644 --- a/tests/pygeoapi-test-ogr-config.yml +++ b/tests/pygeoapi-test-ogr-config.yml @@ -80,9 +80,9 @@ metadata: instructions: During hours of service. Off on weekends. role: pointOfContact -datasets: - +resources: dutch_georef_stations: + type: collection title: Dutch Georef Stations via OGR WFS description: Locations of RD/GNSS-reference stations from Dutch Kadaster PDOK a.k.a RDInfo. Uses MapServer WFS v2 backend via OGRProvider. keywords: @@ -130,6 +130,7 @@ datasets: # Warning: this layer contains about 10 million addresses, the backend WFS seems not optimized dutch_addresses: + type: collection title: Dutch Addresses via OGR WFS description: All Dutch addresses as derived from the key registry BAG. Uses GeoServer WFS v2 backend via OGRProvider. SLOW BACKEND! keywords: @@ -176,6 +177,7 @@ datasets: layer: inspireadressen:inspireadressen utah_city_locations: + type: collection title: Cities in Utah via OGR WFS description: Data from the state of Utah. Standard demo dataset from the deegree WFS server that is used as backend WFS. keywords: @@ -221,6 +223,7 @@ datasets: layer: app:SGID93_LOCATION_UDOTMap_CityLocations unesco_pois_italy: + type: collection title: Unesco POIs in Italy via OGR WFS description: Unesco Points of Interest in Italy. Using GeoSolutions GeoServer WFS demo-server as backend WFS. keywords: @@ -265,6 +268,7 @@ datasets: layer: unesco:Unesco_point ogr_gpkg_poi: + type: collection title: Portuguese Points of Interest via OGR GPKG description: Portuguese Points of Interest obtained from OpenStreetMap. Dataset includes Madeira and Azores islands. Uses GeoPackage backend via OGR provider. keywords: @@ -311,6 +315,7 @@ datasets: sf_311incidents: + type: collection title: SF 311Incidents via OGR ESRI Feature Server description: OGR Provider - ESRI Feature Server - SF 311Incidents keywords: @@ -390,7 +395,7 @@ datasets: time_field: data layer: dpc-covid19-ita-regioni -processes: hello-world: + type: process processor: name: HelloWorld diff --git a/tests/test_api.py b/tests/test_api.py index 8f1bf11..23a767e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -246,7 +246,7 @@ def test_get_collection_queryables(config, api_): assert len(queryables['queryables']) == 6 # test with provider filtered properties - api_.config['datasets']['obs']['provider']['properties'] = ['stn_id'] + api_.config['resources']['obs']['provider']['properties'] = ['stn_id'] rsp_headers, code, response = api_.get_collection_queryables( req_headers, {'f': 'json'}, 'obs') @@ -480,7 +480,7 @@ def test_get_collection_items(config, api_): rsp_headers, code, response = api_.get_collection_items( req_headers, {'datetime': '2002/2014-04-22'}, 'obs') - api_.config['datasets']['obs']['extents'].pop('temporal') + api_.config['resources']['obs']['extents'].pop('temporal') rsp_headers, code, response = api_.get_collection_items( req_headers, {'datetime': '2002/2014-04-22'}, 'obs') @@ -605,7 +605,7 @@ def test_describe_processes(config, api_): assert len(process['outputTransmission']) == 1 assert len(process['jobControlOptions']) == 1 - api_.config['processes'] = {} + api_.config['resources'] = {} req_headers = make_req_headers() rsp_headers, code, response = api_.describe_processes( @@ -613,8 +613,6 @@ def test_describe_processes(config, api_): processes = json.loads(response) assert len(processes['processes']) == 0 - api_.config.pop('processes') - req_headers = make_req_headers() rsp_headers, code, response = api_.describe_processes( req_headers, {}, 'foo') @@ -645,7 +643,7 @@ def test_execute_process(config, api_): assert response['outputs'][0]['value'] == 'test' - api_.config['processes'] = {} + api_.config['resources'] = {} req_headers = make_req_headers() rsp_headers, code, response = api_.execute_process(req_headers, {}, @@ -654,8 +652,6 @@ def test_execute_process(config, api_): response = json.loads(response) assert response['code'] == 'NotFound' - api_.config.pop('processes') - req_headers = make_req_headers() rsp_headers, code, response = api_.execute_process(req_headers, {}, json.dumps(req_body),