Add ESRI Service provider for OGC API - Features (#954)
This commit is contained in:
@@ -71,6 +71,7 @@ jobs:
|
||||
pytest tests/test_csv__formatter.py
|
||||
pytest tests/test_csv__provider.py
|
||||
pytest tests/test_elasticsearch__provider.py
|
||||
pytest tests/test_esri_provider.py
|
||||
pytest tests/test_filesystem_provider.py
|
||||
pytest tests/test_geojson_provider.py
|
||||
pytest tests/test_mongo_provider.py
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# pygeoapi with ESRI Map and Feature Services
|
||||
|
||||
This folder contains the docker-compose configuration necessary to setup an example
|
||||
`pygeoapi` server using a remote ESRI Service endpoint.
|
||||
|
||||
This config is only for example purposes.
|
||||
|
||||
## Hosting features with ArcGIS
|
||||
|
||||
Many ArcGIS layers are hosted as Feature Services. A collection of publically available
|
||||
layers can be found in the [ArcGIS Living Atlas of the World](https://livingatlas.arcgis.com/en/browse/#d=2&q=Feature%20Service).
|
||||
|
||||
The ESRI feature provider creates pygeoapi feature collections from hosted layers. In addition to
|
||||
hosting data from distributed data providers in one place, pygeoapi creates landing pages for
|
||||
individual features in the layer.
|
||||
|
||||
## Building and Running
|
||||
|
||||
To build and run the [Docker compose file](docker-compose.yml) in localhost:
|
||||
|
||||
```
|
||||
docker compose up [--build] [-d]
|
||||
```
|
||||
|
||||
Navigate to `localhost:5000`.
|
||||
@@ -0,0 +1,42 @@
|
||||
# =================================================================
|
||||
#
|
||||
# Authors: Benjamin Webb <bwebb@lincolninst.edu>
|
||||
#
|
||||
# Copyright (c) 2022 Benjamin Webb
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person
|
||||
# obtaining a copy of this software and associated documentation
|
||||
# files (the "Software"), to deal in the Software without
|
||||
# restriction, including without limitation the rights to use,
|
||||
# copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following
|
||||
# conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be
|
||||
# included in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
# OTHER DEALINGS IN THE SOFTWARE.
|
||||
#
|
||||
# =================================================================
|
||||
|
||||
services:
|
||||
pygeoapi:
|
||||
image: geopython/pygeoapi:latest
|
||||
# build:
|
||||
# context: ../../..
|
||||
|
||||
container_name: pygeoapi_esri
|
||||
|
||||
ports:
|
||||
- 5000:80
|
||||
|
||||
volumes:
|
||||
- ./esri.config.yml:/pygeoapi/local.config.yml
|
||||
@@ -0,0 +1,160 @@
|
||||
# =================================================================
|
||||
#
|
||||
# Authors: Benjamin Webb <bwebb@lincolninst.edu>
|
||||
#
|
||||
# Copyright (c) 2022 Benjamin Webb
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person
|
||||
# obtaining a copy of this software and associated documentation
|
||||
# files (the "Software"), to deal in the Software without
|
||||
# restriction, including without limitation the rights to use,
|
||||
# copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following
|
||||
# conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be
|
||||
# included in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
# OTHER DEALINGS IN THE SOFTWARE.
|
||||
#
|
||||
# =================================================================
|
||||
|
||||
server:
|
||||
bind:
|
||||
host: 0.0.0.0
|
||||
port: 80
|
||||
url: http://localhost:5000
|
||||
mimetype: application/json; charset=UTF-8
|
||||
encoding: utf-8
|
||||
gzip: false
|
||||
language: en-US
|
||||
cors: true
|
||||
pretty_print: true
|
||||
limit: 10
|
||||
# templates: /path/to/templates
|
||||
map:
|
||||
url: https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}.png
|
||||
attribution: '<a href="https://wikimediafoundation.org/wiki/Maps_Terms_of_Use">Wikimedia maps</a> | Map data © <a href="https://openstreetmap.org/copyright">OpenStreetMap contributors</a>'
|
||||
ogc_schemas_location: /schemas.opengis.net
|
||||
|
||||
logging:
|
||||
level: ERROR
|
||||
#logfile: /tmp/pygeoapi.log
|
||||
|
||||
metadata:
|
||||
identification:
|
||||
title: ESRI pygeoapi demo instance
|
||||
description: pygeoapi for ESRI Feature and Map Services
|
||||
keywords:
|
||||
- geospatial
|
||||
- esri
|
||||
- api
|
||||
keywords_type: theme
|
||||
terms_of_service: https://creativecommons.org/licenses/by/4.0/
|
||||
url: https://github.com/geopython/pygeoapi
|
||||
license:
|
||||
name: CC-BY 4.0 license
|
||||
url: https://creativecommons.org/licenses/by/4.0/
|
||||
provider:
|
||||
name: Center for Geospatial Solutions
|
||||
url: https://www.lincolninst.edu/center-geospatial-solutions
|
||||
contact:
|
||||
name: Webb, Benjamin
|
||||
position: Softare Developer
|
||||
address: Mailing Address
|
||||
city: City
|
||||
stateorprovince: Administrative Area
|
||||
postalcode: Zip or Postal Code
|
||||
country: Canada
|
||||
phone: +xx-xxx-xxx-xxxx
|
||||
fax: +xx-xxx-xxx-xxxx
|
||||
email: you@example.org
|
||||
url: Contact URL
|
||||
hours: Hours of Service
|
||||
instructions: During hours of service. Off on weekends.
|
||||
role: pointOfContact
|
||||
|
||||
resources:
|
||||
counties:
|
||||
type: collection
|
||||
title: Counties
|
||||
description: USA counties generalized boundaries
|
||||
keywords:
|
||||
- counties
|
||||
- featureserver
|
||||
links:
|
||||
- type: text/html
|
||||
rel: canonical
|
||||
title: data source
|
||||
href: https://www.arcgis.com/home/item.html?id=7566e0221e5646f99ea249a197116605
|
||||
hreflang: en-US
|
||||
extents:
|
||||
spatial:
|
||||
bbox: [-159.8, 19.6, -67.6, 65.5]
|
||||
crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
|
||||
providers:
|
||||
- type: feature
|
||||
name: ESRI
|
||||
data: https://services.arcgis.com/P3ePLMYs2RVChkJx/ArcGIS/rest/services/USA_Counties_Generalized/FeatureServer/0
|
||||
id_field: OBJECTID
|
||||
title_field: NAME
|
||||
|
||||
states:
|
||||
type: collection
|
||||
title: States
|
||||
description: USA states generalized boundaries
|
||||
keywords:
|
||||
- states
|
||||
- featureserver
|
||||
links:
|
||||
- type: text/html
|
||||
rel: canonical
|
||||
title: data source
|
||||
href: https://esri.maps.arcgis.com/home/item.html?id=8c2d6d7df8fa4142b0a1211c8dd66903
|
||||
hreflang: en-US
|
||||
extents:
|
||||
spatial:
|
||||
bbox: [-178.2, 18.9, -66.9, 71.4]
|
||||
crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
|
||||
providers:
|
||||
- type: feature
|
||||
name: ESRI
|
||||
data: https://services.arcgis.com/P3ePLMYs2RVChkJx/ArcGIS/rest/services/USA_States_Generalized_Boundaries/FeatureServer/0
|
||||
id_field: OBJECTID
|
||||
title_field: STATE_NAME
|
||||
|
||||
covid:
|
||||
type: collection
|
||||
title: Covid
|
||||
description: New York Times daily cumulative cases (per 100,000) by county
|
||||
keywords:
|
||||
- covid
|
||||
- mapserver
|
||||
links:
|
||||
- type: text/html
|
||||
rel: canonical
|
||||
title: data source
|
||||
href: https://www.arcgis.com/home/item.html?id=628578697fb24d8ea4c32fa0c5ae1843
|
||||
hreflang: en-US
|
||||
extents:
|
||||
spatial:
|
||||
bbox: [-159.8, 19.6, -67.6, 65.5]
|
||||
crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
|
||||
temporal:
|
||||
begin: 2020-03-20T00:00:00Z
|
||||
end: null
|
||||
providers:
|
||||
- type: feature
|
||||
name: ESRI
|
||||
data: https://services1.arcgis.com/0MSEUqKaxRlEPj5g/arcgis/rest/services/ncov_cases_US/FeatureServer/0
|
||||
id_field: OBJECTID
|
||||
time_field: Last_Update
|
||||
title_field: Combined_Key
|
||||
@@ -20,6 +20,7 @@ parameters.
|
||||
|
||||
CSV,✅/✅,results/hits,❌,❌,❌,✅,❌
|
||||
Elasticsearch,✅/✅,results/hits,✅,✅,✅,✅,✅
|
||||
ESRIFeatureService,✅/✅,results/hits,✅,✅,✅,✅,❌
|
||||
GeoJSON,✅/✅,results/hits,❌,❌,❌,✅,❌
|
||||
MongoDB,✅/❌,results,✅,✅,✅,✅,❌
|
||||
OGR,✅/❌,results/hits,✅,❌,❌,✅,❌
|
||||
@@ -93,6 +94,31 @@ This provider has the support for the CQL queries as indicated in the table abov
|
||||
.. seealso::
|
||||
:ref:`cql` for more details on how to use the Common Query Language to filter the collection with specific queries.
|
||||
|
||||
|
||||
ESRI Feature Service
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
To publish an ESRI `Feature Service <https://enterprise.arcgis.com/en/server/latest/publish-services/windows/what-is-a-feature-service-.htm>`
|
||||
or `Map Service <https://enterprise.arcgis.com/en/server/latest/publish-services/windows/what-is-a-map-service.htm>`
|
||||
specify the URL for the service layer in the ``data`` field.
|
||||
|
||||
* ``id_field`` will often be ``OBJECTID``, ``objectid``, or ``FID``.
|
||||
* If the map or feature service is not shared publicly, the ``username`` and ``password`` fields can be set in the
|
||||
configuration to authenticate into the service.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
providers:
|
||||
- type: feature
|
||||
name: ESRI
|
||||
data: https://sampleserver5.arcgisonline.com/arcgis/rest/services/NYTimes_Covid19Cases_USCounties/MapServer/0
|
||||
id_field: objectid
|
||||
time_field: date_in_your_device_time_zone # Optional time field
|
||||
crs: 4326 # Optional crs (default is ESPG:4326)
|
||||
username: username # Optional ArcGIS username
|
||||
password: password # Optional ArcGIS password
|
||||
|
||||
|
||||
OGR
|
||||
^^^
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ PLUGINS = {
|
||||
'CSV': 'pygeoapi.provider.csv_.CSVProvider',
|
||||
'Elasticsearch': 'pygeoapi.provider.elasticsearch_.ElasticsearchProvider', # noqa
|
||||
'ElasticsearchCatalogue': 'pygeoapi.provider.elasticsearch_.ElasticsearchCatalogueProvider', # noqa
|
||||
'ESRI': 'pygeoapi.provider.esri.ESRIServiceProvider',
|
||||
'GeoJSON': 'pygeoapi.provider.geojson.GeoJSONProvider',
|
||||
'OGR': 'pygeoapi.provider.ogr.OGRProvider',
|
||||
'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider',
|
||||
|
||||
@@ -0,0 +1,351 @@
|
||||
# =================================================================
|
||||
#
|
||||
# Authors: Benjamin Webb <bwebb@lincolninst.edu>
|
||||
#
|
||||
# Copyright (c) 2022 Benjamin Webb
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person
|
||||
# obtaining a copy of this software and associated documentation
|
||||
# files (the "Software"), to deal in the Software without
|
||||
# restriction, including without limitation the rights to use,
|
||||
# copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following
|
||||
# conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be
|
||||
# included in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
# OTHER DEALINGS IN THE SOFTWARE.
|
||||
#
|
||||
# =================================================================
|
||||
|
||||
from copy import deepcopy
|
||||
import json
|
||||
import logging
|
||||
from requests import Session, codes
|
||||
|
||||
from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError,
|
||||
ProviderTypeError, ProviderQueryError)
|
||||
from pygeoapi.util import format_datetime
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
ARCGIS_URL = 'https://www.arcgis.com'
|
||||
GENERATE_TOKEN_URL = 'https://www.arcgis.com/sharing/rest/generateToken'
|
||||
|
||||
|
||||
class ESRIServiceProvider(BaseProvider):
|
||||
"""ESRI Feature/Map Service Provider"""
|
||||
|
||||
def __init__(self, provider_def):
|
||||
"""
|
||||
ESRI Class constructor
|
||||
|
||||
:param provider_def: provider definitions from yml pygeoapi-config.
|
||||
data, id_field, name set in parent class
|
||||
|
||||
:returns: pygeoapi.provider.esri.ESRIServiceProvider
|
||||
"""
|
||||
LOGGER.debug('Logger ESRI Init')
|
||||
|
||||
super().__init__(provider_def)
|
||||
|
||||
self.url = f'{self.data}/query'
|
||||
self.crs = provider_def.get('crs', '4326')
|
||||
self.username = provider_def.get('username')
|
||||
self.password = provider_def.get('password')
|
||||
self.token = None
|
||||
|
||||
self.session = Session()
|
||||
|
||||
self.login()
|
||||
self.get_fields()
|
||||
|
||||
def get_fields(self):
|
||||
"""
|
||||
Get fields of ESRI Provider
|
||||
|
||||
:returns: `dict` of fields
|
||||
"""
|
||||
|
||||
if not self.fields:
|
||||
# Load fields
|
||||
params = {'f': 'pjson'}
|
||||
resp = self.get_response(self.data, params=params)
|
||||
|
||||
if resp.get('error') is not None:
|
||||
msg = 'Connection error: {}'.format(resp['error']['message'])
|
||||
LOGGER.error(msg)
|
||||
raise ProviderConnectionError(msg)
|
||||
|
||||
try:
|
||||
# Verify Feature/Map Service supports required capabilities
|
||||
advCapabilities = resp['advancedQueryCapabilities']
|
||||
assert advCapabilities['supportsPagination'] is True
|
||||
assert advCapabilities['supportsOrderBy'] is True
|
||||
assert 'geoJSON' in resp['supportedQueryFormats']
|
||||
except KeyError:
|
||||
msg = f'Could not access resource {self.data}'
|
||||
LOGGER.error(msg)
|
||||
raise ProviderConnectionError(msg)
|
||||
except AssertionError as err:
|
||||
msg = f'Unsupported Feature/Map Server: {err}'
|
||||
LOGGER.error(msg)
|
||||
raise ProviderTypeError(msg)
|
||||
|
||||
for _ in resp['fields']:
|
||||
self.fields.update({_['name']: {'type': _['type']}})
|
||||
|
||||
return self.fields
|
||||
|
||||
def query(self, offset=0, limit=10, resulttype='results',
|
||||
bbox=[], datetime_=None, properties=[], sortby=[],
|
||||
select_properties=[], skip_geometry=False, q=None, **kwargs):
|
||||
"""
|
||||
ESRI query
|
||||
|
||||
:param offset: starting record to return (default 0)
|
||||
:param limit: number of records to return (default 10)
|
||||
:param resulttype: return results or hit limit (default results)
|
||||
:param bbox: bounding box [minx,miny,maxx,maxy]
|
||||
:param datetime_: temporal (datestamp or extent)
|
||||
:param properties: list of tuples (name, value)
|
||||
:param sortby: list of dicts (property, order)
|
||||
:param select_properties: list of property names
|
||||
:param skip_geometry: bool of whether to skip geometry (default False)
|
||||
:param q: full-text search term(s)
|
||||
|
||||
:returns: `dict` of GeoJSON FeatureCollection
|
||||
"""
|
||||
|
||||
# Default feature collection and request parameters
|
||||
|
||||
params = {
|
||||
'f': 'geoJSON',
|
||||
'outSR': self.crs,
|
||||
'outFields': self._make_fields(select_properties),
|
||||
'where': self._make_where(properties, datetime_)
|
||||
}
|
||||
|
||||
if bbox != []:
|
||||
xmin, ymin, xmax, ymax = bbox
|
||||
params['inSR'] = '4326'
|
||||
params['geometryType'] = 'esriGeometryEnvelope'
|
||||
params['geometry'] = f'{xmin},{ymin},{xmax},{ymax}'
|
||||
|
||||
fc = {
|
||||
'type': 'FeatureCollection',
|
||||
'features': [],
|
||||
'numberMatched': self._get_count(params)
|
||||
}
|
||||
|
||||
if resulttype == 'hits':
|
||||
return fc
|
||||
|
||||
params['orderByFields'] = self._make_orderby(sortby)
|
||||
|
||||
params['returnGeometry'] = 'false' if skip_geometry else 'true'
|
||||
params['resultOffset'] = offset
|
||||
params['resultRecordCount'] = limit
|
||||
|
||||
hits_ = min(limit, fc['numberMatched'])
|
||||
fc['features'] = self._get_all(params, hits_)
|
||||
|
||||
fc['numberReturned'] = len(fc['features'])
|
||||
|
||||
return fc
|
||||
|
||||
def get(self, identifier, **kwargs):
|
||||
"""
|
||||
Query ESRI by id
|
||||
|
||||
:param identifier: feature id
|
||||
|
||||
:returns: dict of single GeoJSON feature
|
||||
"""
|
||||
|
||||
LOGGER.debug(f'Fetching item: {identifier}')
|
||||
params = {
|
||||
'f': 'geoJSON',
|
||||
'outSR': self.crs,
|
||||
'objectIds': identifier,
|
||||
'outFields': self._make_fields()
|
||||
}
|
||||
|
||||
resp = self.get_response(self.url, params=params)
|
||||
LOGGER.debug('Returning item')
|
||||
return resp['features'].pop()
|
||||
|
||||
def login(self):
|
||||
# Generate token from username and password
|
||||
if self.token is None:
|
||||
|
||||
if None in [self.username, self.password]:
|
||||
msg = 'Missing ESRI login information, not setting token'
|
||||
LOGGER.debug(msg)
|
||||
return
|
||||
|
||||
params = {
|
||||
'f': 'pjson',
|
||||
'username': self.username,
|
||||
'password': self.password,
|
||||
'referer': ARCGIS_URL
|
||||
}
|
||||
|
||||
LOGGER.debug('Logging in')
|
||||
with self.session.post(GENERATE_TOKEN_URL, data=params) as r:
|
||||
self.token = r.json().get('token')
|
||||
# https://enterprise.arcgis.com/en/server/latest/administer/windows/about-arcgis-tokens.htm
|
||||
self.session.headers.update({
|
||||
'X-Esri-Authorization': f'Bearer {self.token}'
|
||||
})
|
||||
|
||||
def get_response(self, url, **kwargs):
|
||||
# Form URL for GET request
|
||||
LOGGER.debug('Sending query')
|
||||
with self.session.get(url, **kwargs) as r:
|
||||
|
||||
if r.status_code == codes.bad:
|
||||
LOGGER.error('Bad http response code')
|
||||
raise ProviderConnectionError('Bad http response code')
|
||||
try:
|
||||
return r.json()
|
||||
except json.decoder.JSONDecodeError as err:
|
||||
LOGGER.error(f'Bad response at {self.url}')
|
||||
raise ProviderQueryError(err)
|
||||
|
||||
@staticmethod
|
||||
def _make_orderby(sortby):
|
||||
"""
|
||||
Private function: Make ESRI filter from query properties
|
||||
|
||||
:param sortby: `list` of dicts (property, order)
|
||||
|
||||
:returns: ESRI query `order` clause
|
||||
"""
|
||||
if sortby == []:
|
||||
return None
|
||||
|
||||
__ = {'+': 'ASC', '-': 'DESC'}
|
||||
ret = [f'{_["property"]} {__[_["order"]]}' for _ in sortby]
|
||||
|
||||
return ','.join(ret)
|
||||
|
||||
def _make_fields(self, select_properties=[]):
|
||||
"""
|
||||
Make ESRI out fields clause
|
||||
|
||||
:param select_properties: list of property names
|
||||
|
||||
:returns: ESRI query `outFields` clause
|
||||
"""
|
||||
if self.properties == [] and select_properties == []:
|
||||
return '*'
|
||||
|
||||
if self.properties != [] and select_properties != []:
|
||||
outFields = set(self.properties) & set(select_properties)
|
||||
else:
|
||||
outFields = set(self.properties) | set(select_properties)
|
||||
|
||||
return ','.join(outFields)
|
||||
|
||||
def _make_where(self, properties=[], datetime_=None):
|
||||
"""
|
||||
Make ESRI filter from query properties
|
||||
|
||||
:param properties: `list` of tuples (name, value)
|
||||
:param datetime_: `str` temporal (datestamp or extent)
|
||||
|
||||
:returns: ESRI query `where` clause
|
||||
"""
|
||||
|
||||
if properties == [] and datetime_ is None:
|
||||
return '1 = 1'
|
||||
|
||||
p = []
|
||||
|
||||
if properties != []:
|
||||
|
||||
for (k, v) in properties:
|
||||
if 'String' in self.fields[k]['type']:
|
||||
p.append(f"{k} = '{v}'")
|
||||
else:
|
||||
p.append(f"{k} = {v}")
|
||||
|
||||
if datetime_ is not None:
|
||||
|
||||
def esri_dt(dt):
|
||||
return "TIMESTAMP '{}'".format(
|
||||
format_datetime(dt, '%Y-%m-%d %H:%M:%S')
|
||||
)
|
||||
|
||||
tf = self.time_field
|
||||
if '/' in datetime_:
|
||||
time_start, time_end = datetime_.split('/')
|
||||
if time_start != '..':
|
||||
p.append(f'{tf} >= {esri_dt(time_start)}')
|
||||
if time_end != '..':
|
||||
p.append(f'{tf} <= {esri_dt(time_end)}')
|
||||
else:
|
||||
p.append(f'{tf} = {self.esri_date(datetime_)}')
|
||||
|
||||
return ' AND '.join(p)
|
||||
|
||||
def _get_count(self, params):
|
||||
"""
|
||||
Count number of features from query args
|
||||
|
||||
:param params: `dict` of query params
|
||||
|
||||
:returns: `int` of feature count
|
||||
"""
|
||||
params = deepcopy(params)
|
||||
|
||||
params['returnCountOnly'] = 'true'
|
||||
params['f'] = 'pjson'
|
||||
|
||||
response = self.get_response(self.url, params=params)
|
||||
return response.get('count', 0)
|
||||
|
||||
def _get_all(self, params, hits_):
|
||||
"""
|
||||
Get all features from query args
|
||||
|
||||
:param properties: `dict` of query params
|
||||
:param hits_: `int` of number of features to expect
|
||||
|
||||
:returns: `list` of features
|
||||
"""
|
||||
params = deepcopy(params)
|
||||
|
||||
# Return feature collection
|
||||
features = self.get_response(self.url, params=params).get('features')
|
||||
step = len(features)
|
||||
|
||||
# Query if values are less than expected
|
||||
while len(features) < hits_:
|
||||
LOGGER.debug('Fetching next set of values')
|
||||
params['resultOffset'] += step
|
||||
params['resultRecordCount'] += step
|
||||
|
||||
fs = self.get_response(self.url, params=params).get('features')
|
||||
if len(fs) != 0:
|
||||
features.extend(fs)
|
||||
else:
|
||||
break
|
||||
|
||||
return features
|
||||
|
||||
def __exit__(self, **kwargs):
|
||||
self.session.close()
|
||||
|
||||
def __repr__(self):
|
||||
return f'<ESRIServiceProvider> {self.data}'
|
||||
@@ -0,0 +1,182 @@
|
||||
# =================================================================
|
||||
#
|
||||
# Authors: Benjamin Webb <bwebb@lincolninst.edu>
|
||||
#
|
||||
# Copyright (c) 2022 Benjamin Webb
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person
|
||||
# obtaining a copy of this software and associated documentation
|
||||
# files (the "Software"), to deal in the Software without
|
||||
# restriction, including without limitation the rights to use,
|
||||
# copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the
|
||||
# Software is furnished to do so, subject to the following
|
||||
# conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be
|
||||
# included in all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
# OTHER DEALINGS IN THE SOFTWARE.
|
||||
#
|
||||
# =================================================================
|
||||
|
||||
from datetime import datetime
|
||||
import pytest
|
||||
|
||||
from pygeoapi.provider.esri import ESRIServiceProvider
|
||||
from pygeoapi.util import DATETIME_FORMAT
|
||||
|
||||
TIME_FIELD = 'START_DATE'
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def config():
|
||||
# WATERS Mapping Services
|
||||
# source: EPA Water Mapping Services
|
||||
# URL: https://www.epa.gov/waterdata/waters-mapping-services
|
||||
# License: https://edg.epa.gov/EPA_Data_License.html
|
||||
return {
|
||||
'name': 'ESRI',
|
||||
'type': 'feature',
|
||||
'data': 'https://watersgeo.epa.gov/arcgis/rest/services/OWRAD_NP21/TMDL_NP21/MapServer/0', # noqa
|
||||
'id_field': 'OBJECTID',
|
||||
'time_field': TIME_FIELD
|
||||
}
|
||||
|
||||
|
||||
def test_query(config):
|
||||
p = ESRIServiceProvider(config)
|
||||
|
||||
results = p.query()
|
||||
assert results['features'][0]['id'] == 1
|
||||
assert results['numberReturned'] == 10
|
||||
|
||||
results = p.query(limit=50)
|
||||
assert results['numberReturned'] == 50
|
||||
|
||||
results = p.query(offset=10)
|
||||
assert results['features'][0]['id'] == 11
|
||||
assert results['numberReturned'] == 10
|
||||
|
||||
results = p.query(limit=10)
|
||||
assert len(results['features']) == 10
|
||||
assert results['numberMatched'] == 2496
|
||||
|
||||
results = p.query(limit=10001, resulttype='hits')
|
||||
assert results['numberMatched'] == 2496
|
||||
|
||||
|
||||
def test_geometry(config):
|
||||
p = ESRIServiceProvider(config)
|
||||
|
||||
results = p.query()
|
||||
geometry = results['features'][0]['geometry']
|
||||
assert geometry['coordinates'] == [-71.22138524800965, 43.83429729362349]
|
||||
|
||||
results = p.query(skip_geometry=True)
|
||||
assert results['features'][0]['geometry'] is None
|
||||
|
||||
config['crs'] = 3857
|
||||
p = ESRIServiceProvider(config)
|
||||
results = p.query()
|
||||
geometry = results['features'][0]['geometry']
|
||||
assert geometry['coordinates'] == [-7928328.339400001, 5439835.013800003]
|
||||
|
||||
results = p.query(skip_geometry=True)
|
||||
assert results['features'][0]['geometry'] is None
|
||||
|
||||
|
||||
def test_query_bbox(config):
|
||||
p = ESRIServiceProvider(config)
|
||||
|
||||
bbox = [-109, 37, -102, 41]
|
||||
results = p.query(bbox=bbox)
|
||||
assert results['numberReturned'] == 1
|
||||
|
||||
feature = results['features'][0]
|
||||
assert feature['properties']['GEOGSTATE'] == 'CO'
|
||||
|
||||
x, y = feature['geometry']['coordinates']
|
||||
xmin, ymin, xmax, ymax = bbox
|
||||
assert xmin <= x <= xmax
|
||||
assert ymin <= y <= ymax
|
||||
|
||||
|
||||
def test_query_properties(config):
|
||||
p = ESRIServiceProvider(config)
|
||||
|
||||
results = p.query()
|
||||
assert len(results['features'][0]['properties']) == 26
|
||||
|
||||
# Query by property
|
||||
results = p.query(properties=[('GEOGSTATE', 'CO'), ])
|
||||
assert results['features'][0]['properties']['GEOGSTATE'] == 'CO'
|
||||
|
||||
results = p.query(properties=[('GEOGSTATE', 'CO'), ], resulttype='hits')
|
||||
assert results['numberMatched'] == 1
|
||||
|
||||
# Query for property
|
||||
results = p.query(select_properties=['GEOGSTATE', ])
|
||||
assert len(results['features'][0]['properties']) == 1
|
||||
assert 'GEOGSTATE' in results['features'][0]['properties']
|
||||
|
||||
# Query with configured properties
|
||||
config['properties'] = ['OBJECTID', 'GEOGSTATE', 'CYCLE_YEAR']
|
||||
p = ESRIServiceProvider(config)
|
||||
|
||||
results = p.query()
|
||||
props = results['features'][0]['properties']
|
||||
assert all(p in props for p in config['properties'])
|
||||
assert len(props) == 3
|
||||
|
||||
results = p.query(properties=[('GEOGSTATE', 'CO'), ])
|
||||
assert results['features'][0]['properties']['GEOGSTATE'] == 'CO'
|
||||
|
||||
results = p.query(select_properties=['GEOGSTATE', ])
|
||||
assert len(results['features'][0]['properties']) == 1
|
||||
|
||||
|
||||
def test_query_sortby_datetime(config):
|
||||
|
||||
p = ESRIServiceProvider(config)
|
||||
|
||||
results = p.query(sortby=[{'property': 'CYCLE_YEAR', 'order': '+'}])
|
||||
assert results['features'][0]['properties']['CYCLE_YEAR'] == '1998'
|
||||
|
||||
results = p.query(sortby=[{'property': 'CYCLE_YEAR', 'order': '-'}])
|
||||
assert results['features'][0]['properties']['CYCLE_YEAR'] == '2012'
|
||||
|
||||
def feature_time(r):
|
||||
props = r['features'][0]['properties']
|
||||
timestamp = props[TIME_FIELD]/1000
|
||||
timestamp = datetime.fromtimestamp(timestamp)
|
||||
return timestamp.strftime(DATETIME_FORMAT)
|
||||
|
||||
results = p.query(sortby=[{'property': TIME_FIELD, 'order': '+'}])
|
||||
assert feature_time(results) == '1998-04-01T00:00:00.000000Z'
|
||||
|
||||
results = p.query(sortby=[{'property': TIME_FIELD, 'order': '-'}])
|
||||
assert feature_time(results) == '2012-04-01T00:00:00.000000Z'
|
||||
|
||||
results = p.query(datetime_='../2000-01-01T00:00:00.00Z',
|
||||
sortby=[{'property': TIME_FIELD, 'order': '-'}])
|
||||
assert feature_time(results) == '1998-04-01T00:00:00.000000Z'
|
||||
|
||||
results = p.query(datetime_='2000-01-01T00:00:00.00Z/..',
|
||||
sortby=[{'property': TIME_FIELD, 'order': '+'}])
|
||||
assert feature_time(results) == '2000-04-01T00:00:00.000000Z'
|
||||
|
||||
|
||||
def test_get(config):
|
||||
p = ESRIServiceProvider(config)
|
||||
|
||||
result = p.get(6)
|
||||
assert result['id'] == 6
|
||||
assert result['properties']['GEOGSTATE'] == 'DC'
|
||||
Reference in New Issue
Block a user