Oracle Provider (#1329)

* Added Oracle Provider

* Changed author

* Modified formatting with Black

* Adapt Python Flake8 style

* Adapted line length

* Flake8

* Line length <= 79

* Added Oracle provider into CI/CD

* Changed code style to flake8

* style: tabs to spaces

* style: line length

* style: trailing whitespaces

* Changed dictionary concat to old style

* Fixed skip geometry error.

* Added first set of unit tests

* Deleted whitespaces

* Added Oracle provider documentation

* First version Part 4 (CRUD)

* First version OGC API Feature Part 4 (CRUD)

* Changed style for flake8

* Style: trailing whitespaces

* style: line too long

* style: line too long

* CRUD: Added update

* flake nervt

* CRUD: update + delete

* Added tests + fixed errors

* Updated docs

* Added test_get.. + Error fixing

* Worked reviews in

* Added pull request comments

---------

Co-authored-by: Andreas Kosubek <andreas.kosubek@ama.gv.at>
Co-authored-by: xkosubek <133005275+xkosubek@users.noreply.github.com>

.github/workflows/main.yml

@@ -42,7 +42,27 @@ jobs:
           - python-version: 3.7
     env:
       PYGEOAPI_CONFIG: "$(pwd)/pygeoapi-config.yml"
-
+    
+    services:
+      # Oracle service (label used to access the service container)
+      oracle:
+        # Docker Hub image (feel free to change the tag "latest" to any other available one)
+        image: gvenzl/oracle-xe:latest
+        # Provide passwords and other environment variables to container
+        env:
+          ORACLE_RANDOM_PASSWORD: true
+          APP_USER: geo_test
+          APP_USER_PASSWORD: geo_test
+        # Forward Oracle port
+        ports:
+          - 1521:1521
+        # Provide healthcheck script options for startup
+        options: >-
+          --health-cmd healthcheck.sh
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+    
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
@@ -105,6 +125,7 @@ jobs:
         python3 tests/load_mongo_data.py tests/data/ne_110m_populated_places_simple.geojson
         gunzip < tests/data/hotosm_bdi_waterways.sql.gz | psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test
         psql postgresql://postgres:${{ secrets.DatabasePassword || 'postgres' }}@localhost:5432/test -f tests/data/dummy_data.sql
+        python3 tests/load_oracle_data.py
     - name: run unit tests ⚙️
       env:
         POSTGRESQL_PASSWORD: ${{ secrets.DatabasePassword || 'postgres' }}
@@ -126,6 +147,7 @@ jobs:
         pytest tests/test_ogr_sqlite_provider.py
         pytest tests/test_ogr_wfs_provider.py
         pytest tests/test_openapi.py
+        pytest tests/test_oracle_provider.py
         pytest tests/test_postgresql_provider.py
         pytest tests/test_rasterio_provider.py
         pytest tests/test_sensorthings_provider.py

docs/source/data-publishing/ogcapi-features.rst

@@ -266,6 +266,54 @@ Here `test` is the name of database , `points` is the target collection name.
          data: mongodb://localhost:27017/testdb
          collection: testplaces
 
+.. _Oracle:
+
+Oracle
+^^^^^^
+
+.. note::
+  Requires Python package oracledb
+
+.. code-block:: yaml
+
+  providers:
+      - type: feature
+        name: OracleDB
+        data:
+            host: 127.0.0.1
+            port: 1521 # defaults to 1521 if not provided
+            service_name: XEPDB1
+            # sid: XEPDB1
+            user: geo_test
+            password: geo_test
+            # external_auth: wallet
+            # tns_name: XEPDB1
+            # tns_admin /opt/oracle/client/network/admin 
+            # init_oracle_client: True
+
+        id_field: id
+        table: lakes
+        geom_field: geometry
+        title_field: name
+        # sql_manipulator: tests.test_oracle_provider.SqlManipulator
+        # sql_manipulator_options:
+        #     foo: bar
+        # mandatory_properties:
+        # - bbox
+        # source_crs: 31287 # defaults to 4326 if not provided
+        # target_crs: 31287 # defaults to 4326 if not provided
+
+The provider supports connection over host and port with SID or SERVICE_NAME. For TNS naming, the system 
+environment variable TNS_ADMIN or the configuration parameter tns_admin must be set.
+
+The providers supports external authentication. At the moment only wallet authentication is implemented.
+
+Sometimes it is necessary to use the Oracle client for the connection. In this case init_oracle_client must be set to True.
+
+The provider supports a SQL-Manipulator-Plugin class. With this, the SQL statement could be manipulated. This is
+useful e.g. for authorization at row level or manipulation of the explain plan with hints. 
+
+An example an more informations about that feature you can find in the test class in tests/test_oracle_provider.py.
 
 .. _PostgreSQL:
 

pygeoapi/plugin.py

@@ -50,6 +50,7 @@ PLUGINS = {
         'MapScript': 'pygeoapi.provider.mapscript_.MapScriptProvider',
         'MongoDB': 'pygeoapi.provider.mongo.MongoProvider',
         'MVT': 'pygeoapi.provider.mvt.MVTProvider',
+        'OracleDB': 'pygeoapi.provider.oracle.OracleProvider',
         'OGR': 'pygeoapi.provider.ogr.OGRProvider',
         'PostgreSQL': 'pygeoapi.provider.postgresql.PostgreSQLProvider',
         'rasterio': 'pygeoapi.provider.rasterio_.RasterioProvider',

requirements-provider.txt

@@ -6,6 +6,7 @@ fiona
 #GDAL>=3.0.0
 geoalchemy
 netCDF4
+oracledb
 pandas; python_version < '3.7'
 pandas==1.2.5; python_version >= '3.7'
 psycopg2

tests/test_oracle_provider.py

@@ -0,0 +1,455 @@
+# =================================================================
+#
+# Authors: Andreas Kosubek <andreas.kosubek@ama.gv.at>
+#
+# Copyright (c) 2023 Andreas Kosubek
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+
+# Needs to be run like: python3 -m pytest
+# Create testdata: python3 load_oracle_data.py
+
+import os
+import pytest
+from pygeoapi.provider.oracle import OracleProvider
+
+USERNAME = os.environ.get("PYGEOAPI_ORACLE_USER", "geo_test")
+PASSWORD = os.environ.get("PYGEOAPI_ORACLE_PASSWD", "geo_test")
+SERVICE_NAME = os.environ.get("PYGEOAPI_ORACLE_SERVICE_NAME", "XEPDB1")
+HOST = os.environ.get("PYGEOAPI_ORACLE_HOST", "127.0.0.1")
+PORT = os.environ.get("PYGEOAPI_ORACLE_PORT", "1521")
+
+
+class SqlManipulator:
+    def process_query(
+        self,
+        db,
+        sql_query,
+        bind_variables,
+        sql_manipulator_options,
+        bbox,
+        source_crs,
+        properties,
+    ):
+        sql = "ID = 10 AND :foo != :bar"
+
+        if sql_query.find(" WHERE ") == -1:
+            sql_query = sql_query.replace("#WHERE#", f" WHERE {sql}")
+        else:
+            sql_query = sql_query.replace("#WHERE#", f" AND {sql}")
+
+        bind_variables = {
+            **bind_variables,
+            "foo": "foo",
+            "bar": sql_manipulator_options.get("foo"),
+        }
+
+        return sql_query, bind_variables
+
+    def process_get(
+        self,
+        db,
+        sql_query,
+        bind_variables,
+        sql_manipulator_options,
+        identifier,
+    ):
+        sql_query = f"{sql_query} AND 'auth' = 'you arent allowed'"
+
+        return sql_query, bind_variables
+
+    def process_create(
+        self,
+        db,
+        sql_query,
+        bind_variables,
+        sql_manipulator_options,
+        request_data,
+    ):
+        bind_variables["name"] = "overwritten"
+
+        return sql_query, bind_variables
+
+    def process_update(
+        self,
+        db,
+        sql_query,
+        bind_variables,
+        sql_manipulator_options,
+        identifier,
+        request_data,
+    ):
+        bind_variables["area"] = 42
+        bind_variables["volume"] = 42
+
+        return sql_query, bind_variables
+
+    def process_delete(
+        self,
+        db,
+        sql_query,
+        bind_variables,
+        sql_manipulator_options,
+        identifier,
+    ):
+        sql_query = f"{sql_query} AND 'auth' = 'you arent allowed'"
+
+        return sql_query, bind_variables
+
+
+@pytest.fixture()
+def config():
+    return {
+        "name": "Oracle",
+        "type": "feature",
+        "data": {
+            "host": HOST,
+            "port": PORT,
+            "service_name": SERVICE_NAME,
+            "user": USERNAME,
+            "password": PASSWORD,
+        },
+        "id_field": "id",
+        "table": "lakes",
+        "geom_field": "geometry",
+        "editable": True,
+    }
+
+
+@pytest.fixture()
+def config_manipulator(config):
+    return {
+        **config,
+        "sql_manipulator": "tests.test_oracle_provider.SqlManipulator",
+        "sql_manipulator_options": {"foo": "bar"},
+    }
+
+
+@pytest.fixture()
+def config_properties(config):
+    return {
+        **config,
+        "properties": ["id", "name", "wiki_link"],
+    }
+
+
+@pytest.fixture()
+def create_geojson():
+    return {
+        "type": "Feature",
+        "geometry": {
+            "type": "Polygon",
+            "coordinates": [
+                [
+                    [9.012050, 47.841512],
+                    [9.803470, 47.526461],
+                    [9.476940, 47.459178],
+                    [8.918151, 47.693253],
+                    [9.012050, 47.841512],
+                ]
+            ],
+        },
+        "properties": {
+            "name": "Lake Constance",
+            "wiki_link": "https://en.wikipedia.org/wiki/Lake_Constance",
+            "foo": "bar",
+        },
+    }
+
+
+@pytest.fixture()
+def update_geojson():
+    return {
+        "type": "Feature",
+        "geometry": {
+            "type": "Polygon",
+            "coordinates": [
+                [
+                    [9.012050, 47.841512],
+                    [9.803470, 47.526461],
+                    [9.476940, 47.459178],
+                    [8.918151, 47.693253],
+                    [9.012050, 47.841512],
+                ]
+            ],
+        },
+        "properties": {
+            "name": "Lake Constance",
+            "wiki_link": "https://en.wikipedia.org/wiki/Lake_Constance",
+            "foo": "bar",
+            "area": 536000,
+            "volume": 48000,
+        },
+        "id": 26,
+    }
+
+
+def test_query(config):
+    """Test query for a valid JSON object with geometry"""
+    p = OracleProvider(config)
+    feature_collection = p.query()
+    assert feature_collection.get("type") == "FeatureCollection"
+    features = feature_collection.get("features")
+    assert features is not None
+    feature = features[0]
+    properties = feature.get("properties")
+    assert properties is not None
+    geometry = feature.get("geometry")
+    assert geometry is not None
+
+
+def test_get_fields(config):
+    """Test get_fields"""
+    expected_fields = {
+        "id": {"type": "NUMBER"},
+        "area": {"type": "NUMBER"},
+        "volume": {"type": "NUMBER"},
+        "name": {"type": "VARCHAR2"},
+        "wiki_link": {"type": "VARCHAR2"},
+    }
+
+    provider = OracleProvider(config)
+
+    assert provider.get_fields() == expected_fields
+    assert provider.fields == expected_fields
+
+
+def test_get_fields_properties(config_properties):
+    """
+    Test get_fields with subset of columns.
+    Test of property configuration.
+    """
+    expected_fields = {
+        "id": {"type": "NUMBER"},
+        "name": {"type": "VARCHAR2"},
+        "wiki_link": {"type": "VARCHAR2"},
+    }
+
+    provider = OracleProvider(config_properties)
+    provided_fields = provider.get_fields()
+    print(provided_fields)
+
+    assert provided_fields == expected_fields
+    assert provider.fields == expected_fields
+
+
+def test_query_with_property_filter(config):
+    """Test query valid features when filtering by property"""
+    p = OracleProvider(config)
+    feature_collection = p.query(properties=[("name", "Aral Sea")])
+    features = feature_collection.get("features")
+
+    assert len(features) == 1
+    assert features[0].get("id") == 12
+
+
+def test_query_bbox(config):
+    """Test query with a specified bounding box"""
+    p = OracleProvider(config)
+    feature_collection = p.query(bbox=[50, 40, 60, 50])
+    features = feature_collection.get("features")
+
+    assert len(features) == 1
+    assert features[0]["properties"]["name"] == "Aral Sea"
+
+
+def test_query_sortby(config):
+    """Test query with sorting"""
+    p = OracleProvider(config)
+    up = p.query(sortby=[{"property": "id", "order": "+"}])
+    assert up["features"][0]["id"] == 1
+    down = p.query(sortby=[{"property": "id", "order": "-"}])
+    assert down["features"][0]["id"] == 25
+
+    name = p.query(sortby=[{"property": "name", "order": "+"}])
+    assert name["features"][0]["properties"]["name"] == "Aral Sea"
+    name = p.query(sortby=[{"property": "name", "order": "-"}])
+    assert name["features"][0]["properties"]["name"] == "Vänern"
+
+
+def test_query_skip_geometry(config):
+    """Test query without geometry"""
+    p = OracleProvider(config)
+    result = p.query(skip_geometry=True)
+    feature = result["features"][0]
+
+    assert feature.get("geometry") is None
+
+
+def test_query_hits(config):
+    """Test query number of hits"""
+    p = OracleProvider(config)
+    result = p.query(bbox=[0, 0, 70, 60], resulttype="hits")
+
+    assert result.get("numberMatched") == 5
+
+
+def test_get(config):
+    """Test simple get"""
+    p = OracleProvider(config)
+    result = p.get(5)
+
+    assert result.get("id") == 5
+    assert result.get("prev") == 4
+    assert result.get("next") == 6
+
+
+def test_create(config, create_geojson):
+    """Test simple create"""
+    p = OracleProvider(config)
+    result = p.create(create_geojson)
+
+    assert result == 26
+
+    data = p.get(26)
+
+    assert data.get("properties").get("name") == "Lake Constance"
+
+
+def test_update(config, update_geojson):
+    """Test simple update"""
+    p = OracleProvider(config)
+    identifier = 26
+    result = p.update(identifier, update_geojson)
+
+    assert result
+
+    data = p.get(identifier)
+
+    assert data.get("properties").get("area") == 536000
+    assert data.get("properties").get("volume") == 48000
+
+
+def test_update_properties(config_properties, config, update_geojson):
+    """
+    Test update with filtered columnlist in configuration
+    In this case, the columns area and volume shouldn't be updated!
+    """
+    p = OracleProvider(config_properties)
+    identifier = 26
+
+    update_geojson["properties"]["area"] = 42
+    update_geojson["properties"]["volume"] = 42
+
+    result = p.update(identifier, update_geojson)
+
+    assert result
+
+    p2 = OracleProvider(config)
+    data = p2.get(identifier)
+
+    assert data.get("properties").get("area") == 536000
+    assert data.get("properties").get("volume") == 48000
+
+
+def test_delete(config):
+    """Test simple delete"""
+    p = OracleProvider(config)
+    identifier = 26
+
+    result = p.delete(identifier)
+
+    assert result
+
+    down = p.query(sortby=[{"property": "id", "order": "-"}])
+    assert down["features"][0]["id"] == 25
+
+
+def test_query_sql_manipulator(config_manipulator):
+    """Test SQL manipulator"""
+    p = OracleProvider(config_manipulator)
+    feature_collection = p.query()
+    features = feature_collection.get("features")
+
+    assert len(features) == 1
+    assert features[0].get("id") == 10
+
+
+def test_get_sql_manipulator(config_manipulator):
+    """
+    Test get with SQL manipulator that throws
+    an authorization error.
+    """
+    p = OracleProvider(config_manipulator)
+
+    with pytest.raises(Exception):
+        p.get(5)
+
+
+def test_create_sql_manipulator(config_manipulator, config, create_geojson):
+    """
+    Test create with SQL Manipulator call.
+    Field name should be overwritten with the string "overwritten"
+    """
+    expected_identifier = 27
+
+    p = OracleProvider(config_manipulator)
+    result = p.create(create_geojson)
+
+    assert result == expected_identifier
+
+    p2 = OracleProvider(config)
+    data = p2.get(expected_identifier)
+
+    assert data.get("properties").get("name") == "overwritten"
+
+
+def test_update_sql_manipulator(config_manipulator, config, update_geojson):
+    """
+    Test update with SQL Manipulator call
+    Field names area and volume should be overwritten with the answer to
+    life the universe and everything
+    """
+    identifier = 27
+
+    p = OracleProvider(config_manipulator)
+    result = p.update(identifier, update_geojson)
+
+    assert result
+
+    p2 = OracleProvider(config)
+    data = p2.get(identifier)
+
+    assert data.get("properties").get("area") == 42
+    assert data.get("properties").get("volume") == 42
+
+
+def test_delete_sql_manipulator(config_manipulator, config):
+    """
+    Test for delete with SQL Manipulator call
+    Where clause is overwritten by the manipulator to not
+    match to any record. No record should be deleted.
+    """
+    identifier = 27
+
+    p = OracleProvider(config_manipulator)
+
+    result = p.delete(identifier)
+
+    assert not result
+
+    p2 = OracleProvider(config)
+
+    down = p2.query(sortby=[{"property": "id", "order": "-"}])
+    assert down["features"][0]["id"] == identifier