From c02d8f5ba25e2d68e486fd79f09e5a5e8f49edfe Mon Sep 17 00:00:00 2001 From: Juan Pablo Duque Date: Thu, 27 Apr 2023 11:03:39 +0200 Subject: [PATCH] Fixed Django serve error related to urlpatterns, updated documentation by adding hot-reloading section. (#1231) * Fix for Django build, changes to documentation, and dev dependencies updated * remove example configuration files * Added readthedocs configuration file * added section on building the documentation * renamed docs/requirements-docs.txt to docs/requirements.txt * renamed requirements file in readthedocs configuration * Added documentation requirements installation to GitHub build workflow --- .github/workflows/main.yml | 1 + .gitignore | 1 + .readthedocs.yaml | 21 ++++++ docs/requirements.txt | 4 ++ docs/source/conf.py | 6 +- docs/source/contributing.rst | 41 +++++++++++ docs/source/plugins.rst | 5 +- docs/source/running.rst | 127 ++++++++++++++++++++++++++--------- pygeoapi/django_/urls.py | 10 ++- requirements-dev.txt | 4 -- 10 files changed, 177 insertions(+), 43 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 docs/requirements.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 12e9494..53595f9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -91,6 +91,7 @@ jobs: pip3 install -r requirements-starlette.txt pip3 install -r requirements-dev.txt pip3 install -r requirements-provider.txt + pip3 install -r docs/requirements.txt python3 setup.py install pip3 install --upgrade numpy elasticsearch pip3 install --upgrade numpy "sqlalchemy<2" diff --git a/.gitignore b/.gitignore index 97c5b89..c6256f1 100644 --- a/.gitignore +++ b/.gitignore @@ -115,6 +115,7 @@ ENV/ .DS_Store examples/django/sample_project/db.sqlite3 +pygeoapi/db.sqlite3 # Backup files *~ diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..174f1ea --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,21 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..95f3ef8 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +# Read the docs dependencies +sphinx==5.3.0 +sphinx_rtd_theme==1.1.1 +readthedocs-sphinx-search==0.1.1 \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index d5f4139..9d89652 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,8 +54,12 @@ class Mock(MagicMock): return MagicMock() +# Avoid warning when building the documentation by mocking libraries MOCK_MODULES = ['osgeo', 'psycopg2', 'psycopg2.sql', 'psycopg2.extras', - 'elasticsearch', 'elasticsearch.client.indices'] + 'geoalchemy2', 'elasticsearch', + 'elasticsearch.client.indices', + 'elasticsearch_dsl', 'geoalchemy2.functions', + 'geoalchemy2.shape'] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) PYGEOAPI_HOME = os.path.abspath('../..') diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index d51db6a..bbf17fe 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -3,5 +3,46 @@ Contributing ============ +Building the documentation +-------------------------- + +To build the documentation in pygeoapi we use `Sphinx`_. The documentation is located in the docs folder. + +.. note:: + For the following instructions to work, you must be located in the root folder of pygeoapi. + +Install the dependencies necessary for building the documentation using the following command: + +.. code-block:: bash + + pip3 install -r docs/requirements.txt + +After installing the requirements, build the documentation using the ``sphinx-build`` command: + +.. code-block:: bash + + sphinx-build -M html docs/source docs/build + + +Or using the following ``make`` command: + +.. code-block:: bash + + make -C docs html + +After building the documentation, the folder ``docs/build`` will contain the website generated with the documentation. +Add the folder to a web server or open the file ``docs/build/html/index.html`` file in a web browser to see the contents of the documentation. + +The documentation is hosted on `readthedocs`_. It is automatically generated from the contents of the ``master`` branch on GitHub. + +The file ``.readthedocs.yaml`` contains the configuration of the readthedocs build. Refer to the `readthedocs configuration file`_ documentation for more information. + +Contributing GitHub page +------------------------ + Please see the `Contributing page `_ for information on contributing to the project. + +.. _`Sphinx`: https://www.djangoproject.com +.. _`readthedocs`: https://docs.readthedocs.io/en/stable/index.html +.. _readthedocs configuration file: https://docs.readthedocs.io/en/stable/config-file/v2.html \ No newline at end of file diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 83bfd69..55adaee 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -16,8 +16,11 @@ pygeoapi itself implements numerous formats, data providers and the process func The pygeoapi architecture supports the following subsystems: * data providers + * output formats + * processes + * process manager The core pygeoapi plugin registry can be found in ``pygeoapi.plugin.PLUGINS``. @@ -51,7 +54,7 @@ The following methods are options to connect a plugin to pygeoapi: * Install this Python package onto your system (``python3 setup.py install``). At this point your new package should be in the ``PYTHONPATH`` of your pygeoapi installation * Specify the main plugin class as the ``name`` of the relevant type in the -pygeoapi configuration. For example, for a new vector data provider: + pygeoapi configuration. For example, for a new vector data provider: .. code-block:: yaml diff --git a/docs/source/running.rst b/docs/source/running.rst index 93711fc..3533d1b 100644 --- a/docs/source/running.rst +++ b/docs/source/running.rst @@ -3,21 +3,32 @@ Running ======= -Now we are ready to start up pygeoapi. +Since pygeoapi is a Python API at its core, it can be served via numerous web server scenarios. +So far, pygeoapi can be served via Flask `WSGI`_, Starlette `ASGI`_, and `Django`_. +This section covers how pygeoapi can be run in development environments and in production environments. +For running pygeoapi using docker, refer to the :ref:`running-with-docker` section. -``pygeoapi serve`` ------------------- +Running in development +---------------------- -The ``pygeoapi serve`` command starts up an instance using Flask as the default server. pygeoapi -can be served via Flask `WSGI`_ or Starlette `ASGI`_. +The ``pygeoapi serve`` is the easiest way to run pygeoapi in your own machine. +This command starts a pygeoapi server instance. By default, a Flask server is started, +but Starlette and Django are available as well. -Since pygeoapi is a Python API at its core, it can be served via numerous web server scenarios. +Using the ``--starlette`` or ``--django`` flags will start pygeoapi using the specified server technology. + +It is also advisable to install the development dependencies (contained in the requirements-dev.txt file) for running pygeoapi for +development. To do so, run the following command: + +.. code-block:: bash + + pip3 install -r requirements-dev.txt .. note:: - Changes to either of the pygeoapi or OpenAPI configurations requires a server restart (configurations - are loaded once at server startup for performance). + * Changes to the configuration files of pygeoapi or OpenAPI requires a server restart (configurations are loaded once at server startup for performance). + * Changes to the codebase require a rebuild (i.e., re-running the ``python3 setup.py install`` command). For instructions for running pygeoapi with hot-reloading, refer to the "Hot-reloading" section. Flask WSGI ^^^^^^^^^^ @@ -69,8 +80,13 @@ Starlette is an ASGI implementation which pygeoapi utilizes to communicate with HTTP request <--> Starlette (pygeoapi/starlette_app.py) <--> pygeoapi API (pygeoapi/api.py) +To use Starlette as the web server it is necessary to install its dependencies running the following command: -The Starlette ASGI server can be run as follows: +.. code-block:: bash + + pip3 install -r requirements-starlette.txt + +Then, the Starlette ASGI server can be run as follows: .. code-block:: bash @@ -100,6 +116,75 @@ To integrate pygeoapi as part of another Starlette application: As a result, your application will be available at http://localhost:5000/ and pygeoapi will be available at http://localhost:5000/oapi +Django +^^^^^^ + +`Django`_ is a Python web framework that encourages rapid development and clean, pragmatic design. + +Similarly to Flask and Starlette, Django can be used by pygeoapi to communicate with the core API. + +.. code-block:: bash + + HTTP request <--> Django (pygeoapi/django_app.py) <--> pygeoapi API (pygeoapi/api.py) + +To use Django as a web server it is necessary to install its dependencies running the following command: + +.. code-block:: bash + + pip3 install -r requirements-django.txt + +After Django rependencies is installed, pygeoapi can be run as follows: + +.. code-block:: bash + + pygeoapi serve --django + +As a result, your Django application will be available at http://localhost:5000/. + + +To integrate pygeoapi as part of another Django project in a pluggable it is necessary to add the pygeoapi urls to the +main Django application urls: + +.. code-block:: python + + from django.contrib import admin + from django.urls import path, include + + from pygeoapi.django_pygeoapi import urls as pygeoapi_urls + + urlpatterns = [ + path('admin/', admin.site.urls), + path('sample-project/', include(pygeoapi_urls)), + ] + + +This integration can be seen in the provided example Django project. Refer to `examples/django/sample_project/README.md` +for the integration of pygeoapi with an already exising Django application. + +Hot-reloading +^^^^^^^^^^^^^ + +The ``pygeoapi serve`` uses the current pygeoapi installation. If the installation was performed using the setup command +provided in the :ref:`install` section (``python3 setup.py install``), changes made to the codebase of pygeoapi are not going to be +reflected in the application until a rebuild (i.e., re-running ``python3 setup.py install``). + +By hot-reloading we mean to be able to directly see changes reflected in the application without reinstalling the pygeoapi package or resetting the server. +This is useful for development, as the changes made by developers are easily and rapidly reflected and they can take advantage +of the hot-reloading capabilities that offer each of the web servers available. + +For enabling hot-reloading, install the pygeoapi package using pip (instead of the setup.py script) with the following command: + +.. code-block:: bash + + pip3 install -e . + +.. note:: + This command must be run from the root directory of pygeoapi. + +After the local package is built, you can use the ``pygeoapi serve`` +again and the changes on the codebase will be directly reflected on the running instance. + + Running in production --------------------- @@ -177,11 +262,11 @@ Gunicorn and Flask is simple to run: Gunicorn and Starlette ^^^^^^^^^^^^^^^^^^^^^^ -Running Gunicorn with Starlette requires the `Uvicorn`_ which provides async capabilities along with Gunicorn. +Running Gunicorn with Starlette requires the `Uvicorn`_ library, which provides async capabilities along with Gunicorn. Uvicorn includes a Gunicorn worker class allowing you to run ASGI applications, with all of Uvicorn's performance benefits, while also giving you Gunicorn's fully-featured process management. -is simple to run from the command, e.g: +It is simple to run using the following command: .. code-block:: bash @@ -190,26 +275,6 @@ is simple to run from the command, e.g: .. note:: Uvicorn is as easy to install as ``pip3 install uvicorn`` - -Django -^^^^^^ - -`Django`_ is a Python web framework that encourages rapid development and clean, pragmatic design. Assuming -a Django install/enabled application: - - -.. code-block:: bash - - pygeoapi serve --django - - -To integrate pygeoapi as part of another Django project in a pluggable way the truly impatient developers can -see `examples/django/sample_project/README.md` for a complete Django application. - -As a result, your Django application will be available at http://localhost:5000/ and pygeoapi will be available -at http://localhost:5000/oapi - - Summary ------- diff --git a/pygeoapi/django_/urls.py b/pygeoapi/django_/urls.py index 85c9966..0ca6b87 100644 --- a/pygeoapi/django_/urls.py +++ b/pygeoapi/django_/urls.py @@ -50,7 +50,7 @@ Including another URLconf """ from django.urls import ( - path, + path ) from django.conf import settings from django.conf.urls import include @@ -243,9 +243,7 @@ if url_route_prefix: # Add static URL and optionally add prefix (note: do NOT use django style here) url_static_prefix = settings.API_RULES.get_url_prefix() -urlpatterns.append( - static( - f"{url_static_prefix}{settings.STATIC_URL}", - document_root=settings.STATIC_ROOT - ) +urlpatterns += static( + f"{url_static_prefix}{settings.STATIC_URL}", + document_root=settings.STATIC_ROOT ) diff --git a/requirements-dev.txt b/requirements-dev.txt index fcbb6d6..1b5bc81 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -28,7 +28,3 @@ wheel pyOpenSSL ndg-httpsclient pyasn1 - -# Read the docs -sphinx==3.0.1 -sphinx_rtd_theme