Catch http errors and item not found (#416)

Change class name


Fix flake8


Fix exceptions order


Add missing class name changes


Fix get function for geojson provider


Fix get function for ES provider


Fix ES tests


Fix ES tests


Change word


Fix get function for csv and mongo providers


Fix import


Fix flake8


Fix get function for sqlite/gpkg provider


Fix get function for PG provider


Fix postgres tests


Fix postgres tests


Fix postgres tests

pygeoapi/api.py

@@ -46,7 +46,7 @@ from pygeoapi.log import setup_logger
 from pygeoapi.plugin import load_plugin, PLUGINS
 from pygeoapi.provider.base import (
     ProviderGenericError, ProviderConnectionError, ProviderNotFoundError,
-    ProviderQueryError)
+    ProviderQueryError, ProviderItemNotFoundError)
 from pygeoapi.util import (dategetter, json_serial, render_j2_template,
                            str2bool, TEMPLATES)
 
@@ -875,6 +875,13 @@ class API(object):
             }
             LOGGER.error(err)
             return headers_, 500, json.dumps(exception)
+        except ProviderItemNotFoundError:
+            exception = {
+                'code': 'NotFound',
+                'description': 'identifier not found'
+            }
+            LOGGER.error(exception)
+            return headers_, 404, json.dumps(exception)
         except ProviderQueryError as err:
             exception = {
                 'code': 'NoApplicableCode',

pygeoapi/provider/base.py

@@ -135,6 +135,11 @@ class ProviderQueryError(ProviderGenericError):
     pass
 
 
+class ProviderItemNotFoundError(ProviderGenericError):
+    """provider query error"""
+    pass
+
+
 class ProviderNotFoundError(ProviderGenericError):
     """provider not found error"""
     pass

pygeoapi/provider/csv_.py

@@ -32,7 +32,8 @@ import csv
 import itertools
 import logging
 
-from pygeoapi.provider.base import BaseProvider, ProviderQueryError
+from pygeoapi.provider.base import (BaseProvider, ProviderQueryError,
+                                    ProviderItemNotFoundError)
 
 LOGGER = logging.getLogger(__name__)
 
@@ -145,8 +146,13 @@ class CSVProvider(BaseProvider):
 
         :returns: dict of single GeoJSON feature
         """
-
-        return self._load(identifier=identifier)
+        item = self._load(identifier=identifier)
+        if item:
+            return item
+        else:
+            err = 'item {} not found'.format(identifier)
+            LOGGER.error(err)
+            raise ProviderItemNotFoundError(err)
 
     def __repr__(self):
         return '<CSVProvider> {}'.format(self.data)

pygeoapi/provider/elasticsearch_.py

@@ -34,7 +34,8 @@ from elasticsearch import Elasticsearch, exceptions, helpers
 from elasticsearch.client.indices import IndicesClient
 
 from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError,
-                                    ProviderQueryError)
+                                    ProviderQueryError,
+                                    ProviderItemNotFoundError)
 
 LOGGER = logging.getLogger(__name__)
 
@@ -322,7 +323,7 @@ class ElasticsearchProvider(BaseProvider):
             result = self.es.search(index=self.index_name, body=query)
             if len(result['hits']['hits']) == 0:
                 LOGGER.error(err)
-                return None
+                raise ProviderItemNotFoundError(err)
             LOGGER.debug('Serializing feature')
             feature_ = self.esdoc2geojson(result['hits']['hits'][0])
         except Exception as err:

pygeoapi/provider/geojson.py

@@ -32,7 +32,7 @@ import logging
 import os
 import uuid
 
-from pygeoapi.provider.base import BaseProvider
+from pygeoapi.provider.base import BaseProvider, ProviderItemNotFoundError
 
 LOGGER = logging.getLogger(__name__)
 
@@ -133,8 +133,9 @@ class GeoJSONProvider(BaseProvider):
                 return feature
 
         # default, no match
-        LOGGER.error('feature {} not found'.format(identifier))
-        return None
+        err = 'item {} not found'.format(identifier)
+        LOGGER.error(err)
+        raise ProviderItemNotFoundError(err)
 
     def create(self, new_feature):
         """Create a new feature

pygeoapi/provider/mongo.py

@@ -34,7 +34,7 @@ from pymongo import MongoClient
 from pymongo import GEOSPHERE
 from pymongo import ASCENDING, DESCENDING
 from pymongo.collection import ObjectId
-from pygeoapi.provider.base import BaseProvider
+from pygeoapi.provider.base import BaseProvider, ProviderItemNotFoundError
 
 LOGGER = logging.getLogger(__name__)
 
@@ -149,7 +149,12 @@ class MongoProvider(BaseProvider):
         """
         featurelist, matchcount = self._get_feature_list(
                                     {'_id': ObjectId(identifier)})
-        return featurelist[0] if featurelist else None
+        if featurelist:
+            return featurelist[0]
+        else:
+            err = 'item {} not found'.format(identifier)
+            LOGGER.error(err)
+            raise ProviderItemNotFoundError(err)
 
     def create(self, new_feature):
         """Create a new feature

pygeoapi/provider/ogr.py

@@ -40,7 +40,8 @@ from osgeo import osr as osgeo_osr
 
 from pygeoapi.provider.base import (
     BaseProvider, ProviderGenericError,
-    ProviderQueryError, ProviderConnectionError)
+    ProviderQueryError, ProviderConnectionError,
+    ProviderItemNotFoundError)
 
 LOGGER = logging.getLogger(__name__)
 
@@ -370,7 +371,7 @@ class OGRProvider(BaseProvider):
             layer.SetAttributeFilter("{field} = '{id}'".format(
                 field=self.id_field, id=identifier))
 
-            ogr_feature = self._get_next_feature(layer)
+            ogr_feature = self._get_next_feature(layer, identifier)
             result = self._ogr_feature_to_json(ogr_feature)
 
         except RuntimeError as err:
@@ -379,6 +380,9 @@ class OGRProvider(BaseProvider):
         except ProviderConnectionError as err:
             LOGGER.error(err)
             raise ProviderConnectionError(err)
+        except ProviderItemNotFoundError as err:
+            LOGGER.error(err)
+            raise ProviderItemNotFoundError(err)
         except Exception as err:
             LOGGER.error(err)
             raise ProviderGenericError(err)
@@ -411,14 +415,19 @@ class OGRProvider(BaseProvider):
         class_ = getattr(module, classname)
         self.source_helper = class_(self)
 
-    def _get_next_feature(self, layer):
+    def _get_next_feature(self, layer, feature_id):
         try:
             # Ignore gdal error
             next_feature = _ignore_gdal_error(layer, 'GetNextFeature')
-            if all(val is None for val in next_feature.items().values()):
-                self.gdal.Error(
-                    self.gdal.CE_Failure, 1, "Object properties are all null"
-                )
+            if next_feature:
+                if all(val is None for val in next_feature.items().values()):
+                    self.gdal.Error(
+                        self.gdal.CE_Failure, 1,
+                        "Object properties are all null"
+                    )
+            else:
+                raise ProviderItemNotFoundError(
+                    "item {} not found".format(feature_id))
             return next_feature
         except RuntimeError as gdalerr:
             LOGGER.error(self.gdal.GetLastErrorMsg())

pygeoapi/provider/postgresql.py

@@ -48,7 +48,7 @@ import json
 import psycopg2
 from psycopg2.sql import SQL, Identifier, Literal
 from pygeoapi.provider.base import BaseProvider, \
-    ProviderConnectionError, ProviderQueryError
+    ProviderConnectionError, ProviderQueryError, ProviderItemNotFoundError
 
 from psycopg2.extras import RealDictCursor
 
@@ -353,12 +353,20 @@ class PostgreSQLProvider(BaseProvider):
                 LOGGER.error(err)
                 raise ProviderQueryError()
 
-            row_data = cursor.fetchall()[0]
+            results = cursor.fetchall()
+            row_data = None
+            if results:
+                row_data = results[0]
             feature = self.__response_feature(row_data)
 
-            feature['prev'] = self.get_previous(cursor, identifier)
-            feature['next'] = self.get_next(cursor, identifier)
-            return feature
+            if feature:
+                feature['prev'] = self.get_previous(cursor, identifier)
+                feature['next'] = self.get_next(cursor, identifier)
+                return feature
+            else:
+                err = 'item {} not found'.format(identifier)
+                LOGGER.error(err)
+                raise ProviderItemNotFoundError(err)
 
     def __response_feature(self, row_data):
         """
@@ -369,17 +377,20 @@ class PostgreSQLProvider(BaseProvider):
         :returns: `dict` of GeoJSON Feature
         """
 
-        rd = dict(row_data)
-        feature = {
-            'type': 'Feature'
-        }
-        feature["geometry"] = json.loads(
-            rd.pop('st_asgeojson'))
+        if row_data:
+            rd = dict(row_data)
+            feature = {
+                'type': 'Feature'
+            }
+            feature["geometry"] = json.loads(
+                rd.pop('st_asgeojson'))
 
-        feature['properties'] = rd
-        feature['id'] = feature['properties'].get(self.id_field)
+            feature['properties'] = rd
+            feature['id'] = feature['properties'].get(self.id_field)
 
-        return feature
+            return feature
+        else:
+            return None
 
     def __response_feature_hits(self, hits):
         """Assembles GeoJSON/Feature number

pygeoapi/provider/sqlite.py

@@ -36,7 +36,8 @@ import logging
 import os
 import json
 from pygeoapi.plugin import InvalidPluginError
-from pygeoapi.provider.base import BaseProvider, ProviderConnectionError
+from pygeoapi.provider.base import (BaseProvider, ProviderConnectionError,
+                                    ProviderItemNotFoundError)
 
 LOGGER = logging.getLogger(__name__)
 
@@ -137,17 +138,20 @@ class SQLiteGPKGProvider(BaseProvider):
         :returns: `dict` of GeoJSON Feature
         """
 
-        rd = dict(row_data)  # sqlite3.Row is doesnt support pop
-        feature = {
-            'type': 'Feature'
-        }
-        feature["geometry"] = json.loads(
-            rd.pop('AsGeoJSON({})'.format(self.geom_col))
-            )
-        feature['properties'] = rd
-        feature['id'] = feature['properties'].pop(self.id_field)
+        if row_data:
+            rd = dict(row_data)  # sqlite3.Row is doesnt support pop
+            feature = {
+                'type': 'Feature'
+            }
+            feature["geometry"] = json.loads(
+                rd.pop('AsGeoJSON({})'.format(self.geom_col))
+                )
+            feature['properties'] = rd
+            feature['id'] = feature['properties'].pop(self.id_field)
 
-        return feature
+            return feature
+        else:
+            return None
 
     def __response_feature_hits(self, hits):
         """Assembles GeoJSON/Feature number
@@ -325,7 +329,12 @@ class SQLiteGPKGProvider(BaseProvider):
         row_data = self.cursor.execute(sql_query, (identifier, )).fetchone()
 
         feature = self.__response_feature(row_data)
-        return feature
+        if feature:
+            return feature
+        else:
+            err = 'item {} not found'.format(identifier)
+            LOGGER.error(err)
+            raise ProviderItemNotFoundError(err)
 
     def __repr__(self):
         return '<SQLiteGPKGProvider> {}, {}'.format(self.data, self.table)

tests/test_csv__provider.py

@@ -29,6 +29,7 @@
 
 import pytest
 
+from pygeoapi.provider.base import ProviderItemNotFoundError
 from pygeoapi.provider.csv_ import CSVProvider
 
 
@@ -92,9 +93,14 @@ def test_query(fixture, config):
 
 def test_get(fixture, config):
     p = CSVProvider(config)
-    results = p.get('404')
-    assert results is None
 
     result = p.get('964')
     assert result['id'] == '964'
     assert result['properties']['value'] == '99.9'
+
+
+def test_get_not_existing_item_raise_exception(fixture, config):
+    """Testing query for a not existing object"""
+    p = CSVProvider(config)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get('404')

tests/test_elasticsearch__provider.py

@@ -29,6 +29,7 @@
 
 import pytest
 
+from pygeoapi.provider.base import ProviderItemNotFoundError
 from pygeoapi.provider.elasticsearch_ import ElasticsearchProvider
 
 
@@ -92,9 +93,14 @@ def test_query(config):
 
 def test_get(config):
     p = ElasticsearchProvider(config)
-    results = p.get('404')
-    assert results is None
 
     result = p.get('3413829')
     assert result['id'] == 3413829
     assert result['properties']['ls_name'] == 'Reykjavik'
+
+
+def test_get_not_existing_item_raise_exception(config):
+    """Testing query for a not existing object"""
+    p = ElasticsearchProvider(config)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get('404')

tests/test_geojson_provider.py

@@ -31,6 +31,7 @@
 import json
 import pytest
 
+from pygeoapi.provider.base import ProviderItemNotFoundError
 from pygeoapi.provider.geojson import GeoJSONProvider
 
 
@@ -80,6 +81,15 @@ def test_get(fixture, config):
     assert 'Dinagat' in results['properties']['name']
 
 
+def test_get_not_existing_item_raise_exception(
+    fixture, config
+):
+    """Testing query for a not existing object"""
+    p = GeoJSONProvider(config)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get(-1)
+
+
 def test_delete(fixture, config):
     p = GeoJSONProvider(config)
     p.delete('123-456')
@@ -139,7 +149,8 @@ def test_update_safe_id(fixture, config):
     p.update('123-456', new_feature)
 
     # Don't let the id change, should not exist
-    assert p.get('SOMETHING DIFFERENT') is None
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get('SOMETHING DIFFERENT')
 
     # Should still be at the old id
     results = p.get('123-456')

tests/test_mongo_provider.py

@@ -29,6 +29,7 @@
 
 import pytest
 
+from pygeoapi.provider.base import ProviderItemNotFoundError
 from pygeoapi.provider.mongo import MongoProvider
 
 monogourl = 'mongodb://localhost:27017/testdb'
@@ -100,8 +101,6 @@ def test_query(config):
 def test_get(config):
     p = MongoProvider(config)
     init(p)
-    results = p.get('123456789012345678901234')
-    assert results is None
 
     res = p.query(properties=[['nameascii', 'Reykjavik']])
     result = p.get(res['features'][0]['id'])
@@ -109,6 +108,14 @@ def test_get(config):
     assert 'Reykjavik' in result['properties']['ls_name']
 
 
+def test_get_not_existing_item_raise_exception(config):
+    """Testing query for a not existing object"""
+    p = MongoProvider(config)
+    init(p)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get('123456789012345678901234')
+
+
 def test_get_fields(config):
     p = MongoProvider(config)
     init(p)
@@ -211,7 +218,8 @@ def test_update_safe_id(config):
     p.update(res['features'][0]['id'], updated_feature)
 
     # Don't let the id change, should not exist
-    assert p.get('123456789012345678901234') is None
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get('123456789012345678901234')
 
     # Should still be at the old id
     results = p.get(res['features'][0]['id'])

tests/test_ogr_csv_provider.py

@@ -34,8 +34,11 @@
 import logging
 
 import pytest
+
+from pygeoapi.provider.base import ProviderItemNotFoundError
 from pygeoapi.provider.ogr import OGRProvider
 
+
 LOGGER = logging.getLogger(__name__)
 
 
@@ -85,6 +88,15 @@ def test_get_vsicurl(config_vsicurl_csv):
     assert '11' in result['properties']['codice_regione']
 
 
+def test_get_not_existing_feature_raise_exception(
+    config_vsicurl_csv
+):
+    """Testing query for a not existing object"""
+    p = OGRProvider(config_vsicurl_csv)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get(-1)
+
+
 def test_query_hits_vsicurl(config_vsicurl_csv):
     """Testing query on entire collection for hits"""
 

tests/test_ogr_esrijson_provider.py

@@ -35,8 +35,11 @@ import logging
 import random
 
 import pytest
+
+from pygeoapi.provider.base import ProviderItemNotFoundError
 from pygeoapi.provider.ogr import OGRProvider
 
+
 LOGGER = logging.getLogger(__name__)
 
 
@@ -99,6 +102,15 @@ def test_get_agol(config_ArcGIS_ESRIJSON, config_random_id):
     assert addr_number in result['properties']['fulladdr']
 
 
+def test_get_agol_not_existing_feature_raise_exception(
+    config_ArcGIS_ESRIJSON
+):
+    """Testing query for a not existing object"""
+    p = OGRProvider(config_ArcGIS_ESRIJSON)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get(-1)
+
+
 def test_query_hits_agol(config_ArcGIS_ESRIJSON):
     """Testing query on entire collection for hits"""
 

tests/test_ogr_gpkg_provider.py

@@ -33,8 +33,10 @@ import logging
 
 import pytest
 
+from pygeoapi.provider.base import ProviderItemNotFoundError
 from pygeoapi.provider.ogr import OGRProvider
 
+
 LOGGER = logging.getLogger(__name__)
 
 
@@ -78,6 +80,15 @@ def test_get(config_poi_portugal):
     assert 'cafe' in result['properties']['fclass']
 
 
+def test_get_not_existing_feature_raise_exception(
+    config_poi_portugal
+):
+    """Testing query for a not existing object"""
+    p = OGRProvider(config_poi_portugal)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get(-1)
+
+
 # Testing with GeoPackage files with identical features
 # (all 2481 addresses in Otterlo Netherlands)
 # in different projections.

tests/test_ogr_shapefile_provider.py

@@ -33,6 +33,7 @@ import logging
 
 import pytest
 
+from pygeoapi.provider.base import ProviderItemNotFoundError
 from pygeoapi.provider.ogr import OGRProvider
 
 LOGGER = logging.getLogger(__name__)
@@ -103,6 +104,15 @@ def test_get_4326(config_shapefile_4326):
     assert 'Mosselsepad' in result['properties']['straatnaam']
 
 
+def test_get_not_existing_feature_raise_exception(
+    config_shapefile_4326
+):
+    """Testing query for a not existing object"""
+    p = OGRProvider(config_shapefile_4326)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get(-1)
+
+
 def test_query_hits_28992(config_shapefile_28992):
     """Testing query on entire collection for hits"""
 

tests/test_ogr_sqlite_provider.py

@@ -33,6 +33,7 @@ import logging
 
 import pytest
 
+from pygeoapi.provider.base import ProviderItemNotFoundError
 from pygeoapi.provider.ogr import OGRProvider
 
 LOGGER = logging.getLogger(__name__)
@@ -76,6 +77,15 @@ def test_get_4326(config_sqlite_4326):
     assert 'Mosselsepad' in result['properties']['straatnaam']
 
 
+def test_get_not_existing_feature_raise_exception(
+    config_sqlite_4326
+):
+    """Testing query for a not existing object"""
+    p = OGRProvider(config_sqlite_4326)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get(-1)
+
+
 def test_query_hits_4326(config_sqlite_4326):
     """Testing query on entire collection for hits"""
 

tests/test_ogr_wfs_provider.py

@@ -32,11 +32,14 @@
 # Needs to be run like: python3 -m pytest
 
 import logging
-from pygeoapi.provider.base import ProviderQueryError
 
 import pytest
+
+from pygeoapi.provider.base import (
+    ProviderQueryError, ProviderItemNotFoundError)
 from pygeoapi.provider.ogr import OGRProvider
 
+
 LOGGER = logging.getLogger(__name__)
 
 
@@ -230,6 +233,24 @@ def test_get_gs_with_geojson_output_too_complex_raise_exception(
         p.get(272)
 
 
+def test_get_gs_not_existing_feature_raise_exception(
+    config_GeoServer_WFS
+):
+    """Testing query for a not existing object"""
+    p = OGRProvider(config_GeoServer_WFS)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get(-1)
+
+
+def test_get_ms_not_existing_feature_raise_exception(
+    config_MapServer_WFS
+):
+    """Testing query for a not existing object"""
+    p = OGRProvider(config_MapServer_WFS)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get(-1)
+
+
 def test_query_hits_ms(config_MapServer_WFS):
     """Testing query on entire collection for hits"""
 

tests/test_postgresql_provider.py

@@ -32,6 +32,8 @@
 # Needs to be run like: python3 -m pytest
 
 import pytest
+
+from pygeoapi.provider.base import ProviderItemNotFoundError
 from pygeoapi.provider.postgresql import PostgreSQLProvider
 
 
@@ -120,3 +122,10 @@ def test_get(config):
     assert 'properties' in result
     assert 'id' in result
     assert 'Kanyosha' in result['properties']['name']
+
+
+def test_get_not_existing_item_raise_exception(config):
+    """Testing query for a not existing object"""
+    p = PostgreSQLProvider(config)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get(-1)

tests/test_sqlite_geopackage_provider.py

@@ -34,6 +34,8 @@
 # (Arguments as py.test and set external variables to the correct config path)
 
 import pytest
+
+from pygeoapi.provider.base import ProviderItemNotFoundError
 from pygeoapi.provider.sqlite import SQLiteGPKGProvider
 
 
@@ -149,3 +151,17 @@ def test_get_geopackage(config_geopackage):
     assert 'properties' in result
     assert 'id' in result
     assert 'Académico' in result['properties']['name']
+
+
+def test_get_sqlite_not_existing_item_raise_exception(config_sqlite):
+    """Testing query for a not existing object"""
+    p = SQLiteGPKGProvider(config_sqlite)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get(1234567890)
+
+
+def test_get_geopackage_not_existing_item_raise_exception(config_geopackage):
+    """Testing query for a not existing object"""
+    p = SQLiteGPKGProvider(config_geopackage)
+    with pytest.raises(ProviderItemNotFoundError):
+        p.get(-1)