From 462c641af399c562d5d157772c4c1ee56b853f29 Mon Sep 17 00:00:00 2001 From: sdc50 Date: Thu, 1 Jun 2023 16:37:24 -0600 Subject: [PATCH 01/13] explore optional dependency implementations make all possible dependencies optional tests passing add min env file add additional_url_patterns add oauth2_provider remove warning about user keys Fix pygments lexer warnings in docs build Add install dependency tip box to plot gizmos. black formatting fix lint Removed lingering references to the old developer tools page Add "Starting with Tethys 5.0..." Update tutorial to require installing plotly optional dep notes for Tethys Portal configuration Fixes typo in GRAVATAR_DEFAULT_SECURE setting Replace deprecated Axes setting in docs. Note: settings.py is updated in master docs for social_auth, mf2, and simple_captcha Reviewed manual production configuration docs for optional args REST API and Terms and conditions optional deps JSON Custom setting optional dependency docs MapLayout and pyshp optional dependency. document template tags most optional dependencies documented --- docs/installation.rst | 27 +- docs/installation/application.rst | 12 + .../production/docker/docker_compose.rst | 8 +- .../advanced/images/webanalytics.png | 3 - .../manual/configuration/advanced/lockout.rst | 12 + .../advanced/multi_factor_auth.rst | 13 + .../configuration/advanced/social_auth.rst | 34 +- .../configuration/advanced/webanalytics.rst | 14 +- .../manual/configuration/basic/database.rst | 21 +- docs/tethys_cli/db.rst | 17 + docs/tethys_cli/docker.rst | 8 + docs/tethys_portal.rst | 3 +- docs/tethys_portal/admin_pages.rst | 38 ++ docs/tethys_portal/configuration.rst | 156 ++++++- docs/tethys_portal/developer_tools.rst | 13 - docs/tethys_portal/tethys_users.rst | 68 ++- docs/tethys_sdk/gizmos/bokeh_view.rst | 44 +- docs/tethys_sdk/gizmos/plotly_view.rst | 42 +- docs/tethys_sdk/jobs/condor_job_type.rst | 12 + docs/tethys_sdk/jobs/condor_workflow_type.rst | 12 + docs/tethys_sdk/jobs/dask_job_type.rst | 12 + docs/tethys_sdk/layouts/map_layout.rst | 5 + docs/tethys_sdk/rest_api.rst | 52 ++- docs/tethys_sdk/templating.rst | 11 + docs/tethys_sdk/testing.rst | 12 + .../tethys_services/dataset_services.rst | 26 +- .../tethys_services/persistent_store.rst | 12 + .../geoserver_reference.rst | 2 +- .../thredds_reference.rst | 12 + .../spatial_dataset_services.rst | 20 +- .../spatial_persistent_store.rst | 12 + .../web_processing_services.rst | 45 +- .../part_1/new_app_project.rst | 22 +- .../part_2/file_upload.rst | 6 +- .../google_earth_engine/part_2/rest_api.rst | 73 ++- docs/tutorials/map_layout/data_prep.rst | 2 +- docs/tutorials/map_layout/new_app_project.rst | 18 +- docs/tutorials/thredds/plot_at_location.rst | 32 +- environment.yml | 136 +++--- min_env.yml | 38 ++ .../test_tethys_cli/test_docker_commands.py | 41 +- .../test_tethys_cli/test_gen_commands.py | 116 +---- .../test_tethys_cli/test_install_commands.py | 31 +- .../test_optional_dependencies.py | 41 ++ tethys_apps/admin.py | 32 +- tethys_apps/base/bokeh_handler.py | 8 +- tethys_apps/migrations/0001_initial_40.py | 390 ---------------- tethys_apps/migrations/0001_initial_41.py | 5 - .../migrations/0002_auto_20221130_2305.py | 32 -- tethys_apps/models.py | 10 +- .../templates/tethys_apps/app_base.html | 24 +- tethys_apps/templatetags/app_theme.py | 12 +- tethys_apps/templatetags/humanize.py | 27 +- tethys_apps/utilities.py | 8 +- tethys_cli/cli_helpers.py | 6 +- tethys_cli/db_commands.py | 4 + tethys_cli/docker_commands.py | 17 +- tethys_cli/gen_commands.py | 29 +- tethys_cli/install_commands.py | 17 +- tethys_compute/migrations/0001_initial_40.py | 417 ------------------ tethys_compute/migrations/0001_initial_41.py | 5 - ...2_alter_condorpyjob__remote_input_files.py | 49 -- tethys_compute/models/condor/condor_py_job.py | 6 +- .../models/condor/condor_py_workflow.py | 6 +- .../models/condor/condor_workflow_node.py | 5 +- tethys_compute/models/dask/dask_field.py | 7 +- tethys_compute/models/dask/dask_job.py | 12 +- tethys_compute/models/dask/dask_scheduler.py | 5 +- tethys_compute/views/dask_dashboard.py | 5 +- tethys_config/context_processors.py | 10 +- tethys_gizmos/gizmo_options/bokeh_view.py | 11 +- tethys_gizmos/gizmo_options/plotly_view.py | 6 +- tethys_gizmos/templatetags/tethys_gizmos.py | 7 +- tethys_gizmos/views/gizmos/jobs_table.py | 5 +- tethys_layouts/views/map_layout.py | 19 +- tethys_portal/context_processors.py | 25 ++ tethys_portal/middleware.py | 96 ++-- tethys_portal/optional_dependencies.py | 55 +++ tethys_portal/settings.py | 176 +++++--- .../templates/analytical_body_bottom.html | 2 + .../templates/analytical_body_top.html | 2 + .../templates/analytical_head_bottom.html | 2 + .../templates/analytical_head_top.html | 2 + tethys_portal/templates/base.html | 24 +- tethys_portal/templates/gravatar.html | 7 + tethys_portal/templates/header.html | 7 +- .../templates/oauth2_provider/base.html | 19 + tethys_portal/templates/terms.html | 2 + .../tethys_portal/accounts/login.html | 6 +- .../tethys_portal/accounts/recaptcha.html | 2 + .../templates/tethys_portal/user/base.html | 6 +- .../templates/tethys_portal/user/profile.html | 35 +- .../tethys_portal/user/settings.html | 3 +- tethys_portal/urls.py | 111 +++-- tethys_portal/views/accounts.py | 13 +- tethys_portal/views/psa.py | 209 ++++----- tethys_portal/views/user.py | 59 ++- .../backends/multi_tenant_mixin.py | 6 +- tethys_services/models.py | 37 +- .../templatetags/tethys_services.py | 5 +- tethys_services/utilities.py | 16 +- 101 files changed, 1769 insertions(+), 1688 deletions(-) delete mode 100644 docs/installation/production/manual/configuration/advanced/images/webanalytics.png delete mode 100644 docs/tethys_portal/developer_tools.rst create mode 100644 min_env.yml create mode 100644 tests/unit_tests/test_tethys_portal/test_optional_dependencies.py delete mode 100644 tethys_apps/migrations/0001_initial_40.py delete mode 100644 tethys_apps/migrations/0002_auto_20221130_2305.py delete mode 100644 tethys_compute/migrations/0001_initial_40.py delete mode 100644 tethys_compute/migrations/0002_alter_condorpyjob__remote_input_files.py create mode 100644 tethys_portal/context_processors.py create mode 100644 tethys_portal/optional_dependencies.py create mode 100644 tethys_portal/templates/analytical_body_bottom.html create mode 100644 tethys_portal/templates/analytical_body_top.html create mode 100644 tethys_portal/templates/analytical_head_bottom.html create mode 100644 tethys_portal/templates/analytical_head_top.html create mode 100644 tethys_portal/templates/gravatar.html create mode 100644 tethys_portal/templates/oauth2_provider/base.html create mode 100644 tethys_portal/templates/terms.html create mode 100644 tethys_portal/templates/tethys_portal/accounts/recaptcha.html diff --git a/docs/installation.rst b/docs/installation.rst index f41a3e6ad..269054b28 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -23,7 +23,7 @@ a. To install the ``tethys-platform`` into a new conda environment then run the .. code-block:: bash - conda create -n tethys -c tethysplatform -c conda-forge tethys-platform + conda create -n tethys -c conda-forge tethys-platform .. tip:: @@ -33,9 +33,11 @@ a. To install the ``tethys-platform`` into a new conda environment then run the **Install Development Build** - To install the latest development build of ``tethys-platform`` add the ``tethysplatform/label/dev`` channel to the list of conda channels:: + To install the latest development build of ``tethys-platform`` add the ``tethysplatform/label/dev`` channel to the list of conda channels: - conda create -n tethys -c tethysplatform/label/dev -c tethysplatform -c conda-forge tethys-platform + .. code-block:: bash + + conda create -n tethys -c tethysplatform/label/dev -c conda-forge tethys-platform Alternatively, to install from source refer to the :ref:`developer_installation` docs. @@ -51,7 +53,9 @@ Anytime you want to work with Tethys Platform, you'll need to activate the ``tet 3. Create a :file:`portal_config.yml` File ------------------------------------------ -To add custom configurations such as the database and other local settings you will need to generate a :file:`portal_config.yml` file. To generate a new template :file:`portal_config.yml` run:: +To add custom configurations such as the database and other local settings you will need to generate a :file:`portal_config.yml` file. To generate a new template :file:`portal_config.yml` run: + +.. code-block:: bash tethys gen portal_config @@ -61,20 +65,24 @@ You can customize your settings in the :file:`portal_config.yml` file after you 4. Configure the Tethys Database -------------------------------- -Tethys Platform requires a PostgreSQL database server. There are several options for setting up a DB server: local, docker, or dedicated. For development environments you can use Tethys to create a local server:: +There are several options for setting up a DB server: local, docker, or remote. Tethys Platform uses a local SQLite database by default. For development environments you can use Tethys to create a local server: + +.. code-block:: bash tethys db configure .. note:: - The tethys db command (:ref:`tethys_db_cmd`) will create a local database server in the directory specified by the ``DIR`` setting in the ``DATABASES`` section of the :file:`portal_config.yml` file. If the value of ``DIR`` is a relative path then the database server will be created relative to directory specified by the ``TETHYS_HOME`` environment variable. By default ``TETHYS_HOME`` is at `~/.tethys`. + The tethys db command (:ref:`tethys_db_cmd`) will create a local database file in the location specified by the ``NAME`` setting in the ``DATABASES`` section of the :file:`portal_config.yml` file (by default ``tethys_platform.sqlite``). If the value of ``NAME`` is a relative path then the database file will be created relative to directory specified by the ``TETHYS_HOME`` environment variable. By default ``TETHYS_HOME`` is at `~/.tethys`. - As an alternative to creating a local database server you can also configure a Docker DB server (see :ref:`using_docker`). A local database server is only recommended for development environments. For production environments please refer to :ref:`production_installation`. +For additional options for configuring a database see :ref:`database_configuration` 5. Start the Development Server ------------------------------- -Once you have a database successfully configured you can run the Tethys development server:: +Once you have a database successfully configured you can run the Tethys development server: + +.. code-block:: bash tethys manage start @@ -84,7 +92,7 @@ This will start up a locally running web server. You can access the Tethys Porta You can customize the port that the server is running on by adding the ``-p`` option. - :: + .. code-block:: bash tethys manage start -p 8001 @@ -110,6 +118,7 @@ Related Docs installation/system_requirements tethys_portal/configuration + installation/database_configuration installation/conda installation/application installation/showcase_apps diff --git a/docs/installation/application.rst b/docs/installation/application.rst index d00481329..9fcd7f770 100644 --- a/docs/installation/application.rst +++ b/docs/installation/application.rst @@ -72,6 +72,18 @@ install.yml This file is generated with your application scaffold. Dependencies that are listed in the ``install.yml`` will be installed with conda and will honor the specified channel priority. If there are any dependencies listed in the ``setup.py`` that are not specified in the ``install.yml`` then these packages will be installed with pip as part of the setup process. This file should be committed with your application code in order to aid installation on a Tethys Portal. +.. important:: + + The ``conda`` sections of the ``install.yml`` file require the `conda` library and optionally the `conda-libmamba-solver` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install these libraries using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge conda conda-libmamba-solver + + # pip + pip install conda + .. literalinclude:: resources/example-install.yml :language: yaml diff --git a/docs/installation/production/docker/docker_compose.rst b/docs/installation/production/docker/docker_compose.rst index d79e72b83..e038af829 100644 --- a/docs/installation/production/docker/docker_compose.rst +++ b/docs/installation/production/docker/docker_compose.rst @@ -305,7 +305,7 @@ c. Add the following contents to each ``.env`` file: **db.env** - .. code-block:: env + .. code-block:: docker # Password of the db admin account POSTGRES_PASSWORD=please_dont_use_default_passwords @@ -323,7 +323,7 @@ c. Add the following contents to each ``.env`` file: **thredds.env** - .. code-block:: env + .. code-block:: docker # Password of the TDM admin user TDM_PW=please_dont_use_default_passwords @@ -355,7 +355,7 @@ c. Add the following contents to each ``.env`` file: **web.env** - .. code-block:: env + .. code-block:: docker # Domain name of server should be first in the list if multiple entries added ALLOWED_HOSTS="\"[localhost]\"" @@ -471,7 +471,7 @@ a. Create a :file:`.gitignore` file: b. Add the following contents to the :file:`.gitignore` file to omit the contents of these directories from being tracked: - .. code-block:: gitignore + .. code-block:: text data/ keys/ diff --git a/docs/installation/production/manual/configuration/advanced/images/webanalytics.png b/docs/installation/production/manual/configuration/advanced/images/webanalytics.png deleted file mode 100644 index 38725750a..000000000 --- a/docs/installation/production/manual/configuration/advanced/images/webanalytics.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9e008b4396af02e9341686d28fd10d5dc981d9f049401b908097a8dcc4719e0d -size 34841 diff --git a/docs/installation/production/manual/configuration/advanced/lockout.rst b/docs/installation/production/manual/configuration/advanced/lockout.rst index 07eb0b4fe..0ec90bb60 100644 --- a/docs/installation/production/manual/configuration/advanced/lockout.rst +++ b/docs/installation/production/manual/configuration/advanced/lockout.rst @@ -6,6 +6,18 @@ Lockout (Optional) **Last Updated:** May 2020 +.. important:: + + This feature requires the `django-axes` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `django-axes` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge django-axes + + # pip + pip install django-axes + Tethys Portal includes lockout capabilities to prevent brute-force login attempts. This capability is provided by the `Django Axes `_ add-on for Django. This document describes the different configuration options that are available for lockout capabilities in Tethys Portal. .. image:: ./images/locked_out.png diff --git a/docs/installation/production/manual/configuration/advanced/multi_factor_auth.rst b/docs/installation/production/manual/configuration/advanced/multi_factor_auth.rst index 0b6e0febc..33fa7a3ca 100644 --- a/docs/installation/production/manual/configuration/advanced/multi_factor_auth.rst +++ b/docs/installation/production/manual/configuration/advanced/multi_factor_auth.rst @@ -6,6 +6,19 @@ Multi Factor Authentication (Optional) **Last Updated:** September 2020 +.. important:: + + These settings require the `django-mfa2`, `arrow`, and `isodate` libraries to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install these libraries using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge django-mfa2 arrow isodate + + # pip + pip install django-mfa2 arrow isodate + + Tethys allows you to enable/enforce the use of multi factor authentication through apps such as LastPass Authenticator or Google Authenticator. This capability is provided by `Django MFA2 `_. This tutorial will show you how to enable that functionality. diff --git a/docs/installation/production/manual/configuration/advanced/social_auth.rst b/docs/installation/production/manual/configuration/advanced/social_auth.rst index e8cae6692..553c07a65 100644 --- a/docs/installation/production/manual/configuration/advanced/social_auth.rst +++ b/docs/installation/production/manual/configuration/advanced/social_auth.rst @@ -6,6 +6,18 @@ Single Sign On (Optional) **Last Updated:** December 2020 +.. important:: + + This feature requires the `social-auth-app-django` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `social-auth-app-django` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge social-auth-app-django + + # pip + pip install social-auth-app-django + Tethys Portal supports authenticating users with several social authentication and single sign on providers such as Google, Facebook, and LinkedIn via the OAuth 2.0 method. The social authentication and authorization features have been implemented using the `Python Social Auth `_ module. Social login is disabled by default, because enabling it requires registering your tethys portal instance with each provider. @@ -187,7 +199,7 @@ Facebook b. Press the ``Setup`` button on the tile (or ``Settings`` if setup previously). c. Specify the following for the Valid OAuth Redirect URIs field: - :: + .. code-block:: https:///oauth2/complete/facebook/ @@ -245,7 +257,7 @@ Google As a security precaution, Google will only accept authentication requests from the hosts listed in the ``Authorized JavaScript Origins`` box. Add the domain of your Tethys Portal to the list. Optionally, you may add a localhost domain to the list to be used during testing: - :: + .. code-block:: https:// http://localhost:8000 @@ -258,7 +270,7 @@ Google You also need to provide the callback URI for Google to call once it has authenticated the user. This follows the pattern ``http:///oauth2/complete/google-oauth2/``: - :: + .. code-block:: https:///oauth2/complete/google-oauth2/ https://localhost:8000/oauth2/complete/google-oauth2/ @@ -299,6 +311,18 @@ For more detailed information about using Google social authentication see the f HydroShare ---------- +.. important:: + + This feature requires the `hs_restclient` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `hs_restclient` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge hs_restclient + + # pip + pip install hs_restclient + 1. Create a HydroShare Account You will need a HydroShare account to register your Tethys Portal with HydroShare. To create an account, visit `https://www.hydroshare.org `_. @@ -319,7 +343,7 @@ HydroShare g. Redirect uris: Add the call back URLs. The protocol (http or https) that matches your Tethys Portal settings should be included in this url. For example: - :: + .. code-block:: if your Tethys Portal was located at the domain ``https://www.my-tethys-portal.com``: https://www.my-tethys-portal.com/oauth2/complete/hydroshare/ @@ -509,7 +533,7 @@ LinkedIn a. Add the call back URLs under the **OAuth 2.0 settings** section: - :: + .. code-block:: https:///oauth2/complete/linkedin-oauth2/ http://localhost:8000/oauth2/complete/linkedin-oauth2/ diff --git a/docs/installation/production/manual/configuration/advanced/webanalytics.rst b/docs/installation/production/manual/configuration/advanced/webanalytics.rst index ecf19ba16..4a571b76e 100644 --- a/docs/installation/production/manual/configuration/advanced/webanalytics.rst +++ b/docs/installation/production/manual/configuration/advanced/webanalytics.rst @@ -4,9 +4,17 @@ Web Analytics (Optional) **Last Updated:** May 2020 -.. image:: ./images/webanalytics.png - :width: 300px - :align: right +.. important:: + + This feature requires the `django-analytical` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `django-analytical` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge django-analytical + + # pip + pip install django-analytical Tethys portals are configured to allow portal administrators to track how users interact with their portal and applications using web based analytical services. 24 services, including common services like Google Analytical and Optimizely, can be configured using the `Django-Analytical `_ package. diff --git a/docs/installation/production/manual/configuration/basic/database.rst b/docs/installation/production/manual/configuration/basic/database.rst index c229e2b85..8c70f71ef 100644 --- a/docs/installation/production/manual/configuration/basic/database.rst +++ b/docs/installation/production/manual/configuration/basic/database.rst @@ -4,11 +4,22 @@ Production Database ******************* -**Last Updated:** September 2022 +**Last Updated:** September 2023 In this part of the production deployment guide, you will learn how to initialize and configure the Tethys Portal database for production. -1. Set Database Settings +1. Install Python dependencies +============================== + +Using a PostgreSQL database for production requires the ``psycopg2`` Python package. Also, While we do not recommend having your database on the same server as Tethys Portal, the commands to automate setting up and configuring the database require that the PostgreSQL database and the `psycopg2` library be installed on the web server. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `postgresql` and `psycopg2` using conda as follows: + + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge postgresql psycopg2 + +2. Set Database Settings ======================== Set the database settings in the :file:`portal_config.yml` using the ``tethys settings`` command: @@ -25,7 +36,7 @@ Set the database settings in the :file:`portal_config.yml` using the ``tethys se **DO NOT USE DEFAULT USERNAMES OR PASSWORDS FOR PRODUCTION DATABASE ACCOUNTS** -2. Create Tethys Database and Database Users +3. Create Tethys Database and Database Users ============================================ Use the ``tethys db create`` command to create the database users and tables required by Tethys Portal: @@ -42,7 +53,7 @@ Use the ``tethys db create`` command to create the database users and tables req **DO NOT USE DEFAULT USERNAMES OR PASSWORDS FOR PRODUCTION DATABASE ACCOUNTS** -3. Create Tethys Database Tables +4. Create Tethys Database Tables ================================ Run the following command to create the Tethys database tables: @@ -51,7 +62,7 @@ Run the following command to create the Tethys database tables: tethys db migrate -4. Create Portal Admin User +5. Create Portal Admin User =========================== You will need to create at least one Portal Admin account to allow you to login to your Tethys Portal. Create the account as follows: diff --git a/docs/tethys_cli/db.rst b/docs/tethys_cli/db.rst index ef4f6950d..7bf3a1eb1 100644 --- a/docs/tethys_cli/db.rst +++ b/docs/tethys_cli/db.rst @@ -5,6 +5,23 @@ db command Setup and manage a Tethys database. +.. important:: + + The default database configuration uses SQLite. Many of these commands are not applicable to SQLite databases are only support PostgreSQL databases. To use a PostgreSQL database be sure to set your settings accordingly. At the very least: + + .. code-block:: bash + + tethys settings --set DATABASES.default.ENGINE django.db.backends.postgresql + + For more details see :ref:`database_configuration` + + Additionally, the PostgreSQL database and the `psycopg2` library be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `postgresql` and `psycopg2` using conda as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge postgresql psycopg2 + .. argparse:: :module: tethys_cli :func: tethys_command_parser diff --git a/docs/tethys_cli/docker.rst b/docs/tethys_cli/docker.rst index 2b8926807..89f7fe03c 100644 --- a/docs/tethys_cli/docker.rst +++ b/docs/tethys_cli/docker.rst @@ -9,6 +9,14 @@ Manage Tethys-sponsored Docker containers. To learn more about Docker, see `What You must have Docker installed and add your user to the ``docker`` group to use the Tethys ``docker`` command (see: `Install Docker `_ and `Post-installation steps for Linux `_). + Additionally, this feature requires the `docker-py` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `docker-py` using conda as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge docker-py + + .. argparse:: :module: tethys_cli :func: tethys_command_parser diff --git a/docs/tethys_portal.rst b/docs/tethys_portal.rst index 20e6768b1..f0c4331c2 100644 --- a/docs/tethys_portal.rst +++ b/docs/tethys_portal.rst @@ -2,7 +2,7 @@ Tethys Portal ************* -**Last Updated:** April 29, 2019 +**Last Updated:** August 2023 Tethys Portal is the Django web site provided by Tethys Platform that acts as the runtime environment for apps. It leverages the capabilities of Django to provide the core website functionality that is often taken for granted in modern web applications. A description of the primary capabilities of Tethys Portal is provided in this section. @@ -12,7 +12,6 @@ Tethys Portal is the Django web site provided by Tethys Platform that acts as th tethys_portal/configuration tethys_portal/admin_pages tethys_portal/tethys_users - tethys_portal/developer_tools tethys_portal/feedback diff --git a/docs/tethys_portal/admin_pages.rst b/docs/tethys_portal/admin_pages.rst index e18197518..6852cb573 100644 --- a/docs/tethys_portal/admin_pages.rst +++ b/docs/tethys_portal/admin_pages.rst @@ -26,6 +26,18 @@ Tethys Portal includes administration pages that can be used to manage the websi Auth Token ========== +.. important:: + + This feature requires the `djangorestframework` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `djangorestframework` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge djangorestframework + + # pip + pip install djangorestframework + Tethys REST API tokens for individual users can be managed using the ``Tokens`` link under the ``AUTH TOKEN`` heading (see Figure 2). .. figure:: ../images/site_admin/auth_token.png @@ -154,6 +166,18 @@ Tethys leverages the excellent `Python Social Auth `_ setting. Defaults to 900 seconds. ================================================== ================================================================================ +.. _database_settings: + DATABASES +++++++++ @@ -116,7 +131,7 @@ See the Django `DATABASES `_ setting. Not used with SQLite. | +--+----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| | DIR | name of psql directory for conda installation of PostgreSQL that ships with Tethys (if using the ``django.db.backends.postgresql`` ``ENGINE``). This directory will be created relative to the ``TETHYS_HOME`` directory when ``tethys db create`` is executed, unless an absolute path is provided. Defaults to ``psql``. If you are using the ``sqlite3`` ``ENGINE`` or an external database server then exclude this key or set it to `None`. | +| | DIR | name of psql directory for conda installation of PostgreSQL that ships with Tethys (if using the ``django.db.backends.postgresql`` ``ENGINE``). This directory will be created relative to the ``TETHYS_HOME`` directory when ``tethys db create`` is executed, unless an absolute path is provided. If you are using the ``sqlite3`` ``ENGINE`` or an external database server then exclude this key or set it to `None`. | +--+----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ LOGGING @@ -153,9 +168,26 @@ LOGGING CAPTCHA_CONFIG ++++++++++++++ +.. important:: + + These Captcha feature requires either the `django-simple-captcha` library or the `django-recaptcha2` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install one of these libraries using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge django-simple-captcha + # Or + conda install -c conda-forge django-recaptcha2 + + # pip + pip install django-simple-captcha + # Or + pip install django-recaptcha2 + ================================================== ================================================================================ Setting Description ================================================== ================================================================================ +ENABLE_CAPTCHA Boolean specifying if captcha should be enabled on the login screen. If using Google ReCaptcha then the following two settings are required. Default is ``False`` RECAPTCHA_PRIVATE_KEY Private key for Google ReCaptcha. Required to enable ReCaptcha on the login screen. See `Django Recaptcha 2 Installation `_. RECAPTCHA_PUBLIC_KEY Public key for Google ReCaptcha. Required to enable ReCaptcha on the login screen. See `Django Recaptcha 2 Installation `_. RECAPTCHA_PROXY_HOST Proxy host for Google ReCaptcha. Optional. See `Django Recaptcha 2 Installation `_. @@ -164,6 +196,28 @@ RECAPTCHA_PROXY_HOST Proxy host for Google ReCaptc OAUTH_CONFIG ++++++++++++ +.. important:: + + These settings require the `social-auth-app-django` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `social-auth-app-django` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge social-auth-app-django + + # pip + pip install social-auth-app-django + + If using the OneLogin OIDC provider, you will also need to install the `python-jose` library: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge python-jose + + # pip + pip install python-jose + ================================================== ================================================================================ Setting Description ================================================== ================================================================================ @@ -214,6 +268,18 @@ SOCIAL_AUTH_ONELOGIN_OIDC_SUBDOMAIN Your OneLogin Subdomain. See MFA_CONFIG ++++++++++ +.. important:: + + These settings require the `django-mfa2`, `arrow`, and `isodate` libraries to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install these libraries using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge django-mfa2 arrow isodate + + # pip + pip install django-mfa2 arrow isodate + ================================================== ================================================================================ Setting Description ================================================== ================================================================================ @@ -230,7 +296,20 @@ MFA_UNALLOWED_METHODS A list of MFA methods to be d ANALYTICS_CONFIG ++++++++++++++++ -the Django Analytical configuration settings for enabling analytics services on the Tethys Portal (see: `Enabling Services - Django Analytical `_. The following is a list of settings for some of the supported services that can be enabled. +The Django Analytical configuration settings for enabling analytics services on the Tethys Portal (see: `Enabling Services - Django Analytical `_. The following is a list of settings for some of the supported services that can be enabled. + + +.. important:: + + These settings require the `django-analytical` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `django-analytical` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge django-analytical + + # pip + pip install django-analytical ================================================== ================================================================================ Setting Description @@ -278,14 +357,26 @@ EMAIL_FROM the email alias setting (e.g. LOCKOUT_CONFIG ++++++++++++++ -the Django Axes configuration settings for enabling lockout capabilities on Tethys Portal (see: :ref:`advanced_config_lockout`). The following is a list of the Django Axes settings that are configured for the default lockout capabilities in Tethys Portal. For a full list of Django Axes settings, see: `Django Axes Configuration Documentation `_. +The Django Axes configuration settings for enabling lockout capabilities on Tethys Portal (see: :ref:`advanced_config_lockout`). The following is a list of the Django Axes settings that are configured for the default lockout capabilities in Tethys Portal. For a full list of Django Axes settings, see: `Django Axes Configuration Documentation `_. + +.. important:: + + These settings require the `django-axes` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `django-axes` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge django-axes + + # pip + pip install django-axes ================================================== ================================================================================ Setting Description ================================================== ================================================================================ AXES_FAILURE_LIMIT Number of failed login attempts to allow before locking. Default ``3``. AXES_COOLOFF_TIME Time to elapse before locked user is allowed to attempt logging in again. In the :file:`portal_config.yml` this setting accepts only integers or `ISO 8601 time duration formatted strings `_ (e.g.: ``"PT30M"``). Default is 30 minutes. -AXES_ONLY_USER_FAILURES Only lock based on username and do not lock based on IP when True. Defaults to ``True``. +AXES_LOCKOUT_PARAMETERS A list of parameters that Axes uses to lock out users. See `Django Axes - Customizing lockout parameters `_ for more details. Defaults to ``['username']``. AXES_ENABLE_ADMIN Enable the Django Axes admin interface. Defaults to ``True``. AXES_VERBOSE More logging for Axes when True. Defaults to ``True``. AXES_RESET_ON_SUCCESS Successful login (after the cooloff time has passed) will reset the number of failed logins when True. Defaults to ``True``. @@ -293,6 +384,61 @@ AXES_LOCKOUT_TEMPLATE Template to render when user AXES_LOGGER The logger for Django Axes to use. Defaults to ``'tethys.watch_login'``. ================================================== ================================================================================ +CORS_CONFIG ++++++++++++ + +These CORS settings are used to configure Cross-Origin Resource Sharing (CORS) for the Tethys Portal. See: `Django CORS Headers `_ for more information for the complete list of availalbe settings. + +.. important:: + + These settings require the `django-cors-headers` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `django-cors-headers` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge django-cors-headers + + # pip + pip install django-cors-headers + +================================================== ================================================================================ +Setting Description +================================================== ================================================================================ +CORS_ALLOWED_ORIGINS A list of origins that are authorized to make cross-site HTTP requests. Defaults to ``[]``. +CORS_ALLOWED_ORIGIN_REGEXES A list of strings representing regexes that match Origins that are authorized to make cross-site HTTP requests. Defaults to ``[]``. +CORS_ALLOW_ALL_ORIGINS If ``True``, all origins will be allowed. Other settings restricting allowed origins will be ignored. Defaults to ``False``. +CORS_ALLOW_METHODS A list of HTTP verbs that are allowed for cross-site requests. Defaults to ``("DELETE", "GET", "OPTIONS", "PATCH", "POST", "PUT")``. +CORS_ALLOW_HEADERS The list of non-standard HTTP headers that you permit in requests from the browser. Sets the Access-Control-Allow-Headers header in responses to preflight requests. Defaults to ``("accept", "authorization", "content-type", "user-agent", "x-csrftoken", "x-requested-with")``. +================================================== ================================================================================ + +Gravatar Settings ++++++++++++++++++ + +The Gravatar settings are used to configure the Gravatar service user profile pictures for the Tethys Portal. See: `Django Gravatar 2 `_ for more information. + +.. important:: + + These settings require the `django-gravatar2` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `django-gravatar2` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge django-gravatar2 + + # pip + pip install django-gravatar2 + +================================================== ================================================================================ +Setting Description +================================================== ================================================================================ +GRAVATAR_URL the Gravatar service endpoint. Defaults to ``"http://www.gravatar.com/"``. +GRAVATAR_SECURE_URL the secure Gravatar service endpoint. Defaults to ``"https://secure.gravatar.com/"``. +GRAVATAR_DEFAULT_SIZE the default size in pixels of the Gravatar image. Defaults to ``"80"``. +GRAVATAR_DEFAULT_IMAGE the default Gravatar image. Defaults to ``"retro"``. +GRAVATAR_DEFAULT_RATING the default allowable image rating. Defaults to ``"g"``. +GRAVATAR_DEFAULT_SECURE uses Gravatar secure endpoint when ``True``. Defaults to ``True``. +================================================== ================================================================================ + Other Settings ++++++++++++++ diff --git a/docs/tethys_portal/developer_tools.rst b/docs/tethys_portal/developer_tools.rst deleted file mode 100644 index 305ca6999..000000000 --- a/docs/tethys_portal/developer_tools.rst +++ /dev/null @@ -1,13 +0,0 @@ -*************** -Developer Tools -*************** - -**Last Updated:** August 4, 2015 - -Tethys provides a Developer Tools page that is accessible when you run Tethys in developer mode. Developer Tools contain documentation, code examples, and live demos of the features of various features of Tethys. Use it to learn how to add a map or a plot to your web app using Gizmos or browse the available geoprocessing capabilities and formulate geoprocessing requests interactively. - -.. figure:: ../images/features/developer_tools.png - :width: 500px - - -**Figure 4.** Use the Developer Tools page to assist you in development. \ No newline at end of file diff --git a/docs/tethys_portal/tethys_users.rst b/docs/tethys_portal/tethys_users.rst index 8e0175df1..4c34f2c04 100644 --- a/docs/tethys_portal/tethys_users.rst +++ b/docs/tethys_portal/tethys_users.rst @@ -15,6 +15,18 @@ The User Settings page can be accessed through the drop-down menu located at the A non-editable view of the user's information can be accessed by clicking the user avatar icon to the left of the drop-down menu (see Figure 1). +.. note:: + + This icon next to the users name come from `Gravatar `_. This feature requires the `django-gravatar2` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `django-gravatar2` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c django-gravatar2 + + # pip + pip install django-gravatar2 + .. figure:: ../images/tethys_portal/tethys_portal_user_profile.png :width: 675px @@ -39,4 +51,58 @@ Within a user's settings page there is a ``Workspace`` section that provides a s .. tip:: - See :ref:`tethys_quotas_workspace_manage` for information on how to pre/post process the user workspace when it is cleared. \ No newline at end of file + See :ref:`tethys_quotas_workspace_manage` for information on how to pre/post process the user workspace when it is cleared. + +Customization +============= + +The Tethys User Profile and Settings pages can be customized by overriding the template used to render them (see the ``Custom Templates`` section in :ref:`tethys_configuration`). + +When providing a custom template you may just want to extend the default template and override specific blocks. For example: + +.. code-block:: html+django + + {% extends "tethys_portal/user/profile.html" %} + + {% block api_key_override %} + {% endblock %} + + {% block custom_sections %} +
+
+

Custom Section

+
+
+
+
{{ custom_user_attribute }}
+
+
+
+
+ {% endblock %} + +The following blocks are defined in the ``profile.html`` file: + +- ``title`` +- ``back_button`` +- ``secondary_content`` + - ``profile_sections`` + - ``name_override`` + - ``name_parameters`` + - ``email_override`` + - ``email_parameters`` + - ``credentials_override`` + - ``credential_parameters`` + - ``sso_override`` + - ``social_parameters`` + - ``api_key_override`` + - ``account_override`` + - ``account_parameters`` + - ``workspace_override`` + - ``storage_parameters`` + - ``oauth2_provider_override`` + - ``custom_sections`` + +.. note:: + + The ``settings.html`` file is what is shown when the user selects the ``Edit`` button on the user profile page. It just extends the ``profile.html`` file and overrides the ``*_parameters`` blocks. diff --git a/docs/tethys_sdk/gizmos/bokeh_view.rst b/docs/tethys_sdk/gizmos/bokeh_view.rst index 9105d8a3a..a9ccfa59c 100644 --- a/docs/tethys_sdk/gizmos/bokeh_view.rst +++ b/docs/tethys_sdk/gizmos/bokeh_view.rst @@ -2,7 +2,24 @@ Bokeh View ********** -**Last Updated:** November 11, 2016 +**Last Updated:** August 2023 + +.. important:: + + This gizmo requires the `bokeh` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `bokeh` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge "bokeh<3" + + # pip + pip install "bokeh<3" + + **Don't Forget**: If you end up using this gizmo in your app, add `bokeh` as a requirement to your `install.yml`. + +Python +------ .. autoclass:: tethys_sdk.gizmos.BokehView @@ -21,7 +38,8 @@ to do so with Bokeh. in the ``import_gizmos`` block. For example: - :: + + .. code-block:: html+django {% block import_gizmos %} {% import_gizmo_dependency bokeh_view %} @@ -30,12 +48,15 @@ to do so with Bokeh. Four elements are required: 1) A controller for the AJAX call with a BokehView gizmo. -:: + + +.. code-block:: python from tethys_sdk.gizmos import BokehView + from tethys_sdk.routing import controller from bokeh.plotting import figure - @login_required() + @controller(name="bokeh_ajax", url="app-name/bokeh") def bokeh_ajax(request): """ Controller for the bokeh ajax request. @@ -49,23 +70,16 @@ Four elements are required: return render(request, 'app_name/bokeh_ajax.html', context) 2) A template for with the tethys gizmo (e.g. bokeh_ajax.html) -:: + +.. code-block:: html+django {% load tethys_gizmos %} {% gizmo bokeh_view_input %} -3) A url map to the controller in app.py -:: - - ... - UrlMap(name='bokeh_ajax', - url='app_name/bokeh', - controller='app_name.controllers.bokeh_ajax'), - ... +3) The AJAX call in the javascript -4) The AJAX call in the javascript -:: +.. code-block:: javascript $(function() { //wait for page to load diff --git a/docs/tethys_sdk/gizmos/plotly_view.rst b/docs/tethys_sdk/gizmos/plotly_view.rst index b48fd52fe..76e378721 100644 --- a/docs/tethys_sdk/gizmos/plotly_view.rst +++ b/docs/tethys_sdk/gizmos/plotly_view.rst @@ -4,7 +4,25 @@ Plotly View *********** -**Last Updated:** November 11, 2016 +**Last Updated:** August 2023 + + +.. important:: + + This gizmo requires the `plotly` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `plotly` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge plotly + + # pip + pip install plotly + + **Don't Forget**: If you end up using this gizmo in your app, add `plotly` as a requirement to your `install.yml`. + +Python +------ .. autoclass:: tethys_sdk.gizmos.PlotlyView @@ -31,13 +49,16 @@ to do so with PlotlyView. Four elements are required: 1) A controller for the AJAX call with a PlotlyView gizmo. -:: + +.. code-block:: python from datetime import datetime import plotly.graph_objs as go from tethys_sdk.gizmos import PlotlyView + from tethys_sdk.routing import controller + - @login_required() + @controller(name='plotly_ajax', url='app-name/plotly') def plotly_ajax(request): """ Controller for the plotly ajax request. @@ -53,23 +74,16 @@ Four elements are required: return render(request, 'app_name/plotly_ajax.html', context) 2) A template for with the tethys gizmo (e.g. plotly_ajax.html) -:: + +.. code-block:: html+django {% load tethys_gizmos %} {% gizmo plotly_view_input %} -3) A url map to the controller in app.py -:: - - ... - UrlMap(name='plotly_ajax', - url='app_name/plotly', - controller='app_name.controllers.plotly_ajax'), - ... +3) The AJAX call in the javascript -4) The AJAX call in the javascript -:: +.. code-block:: javascript $(function() { //wait for page to load diff --git a/docs/tethys_sdk/jobs/condor_job_type.rst b/docs/tethys_sdk/jobs/condor_job_type.rst index a95f51fc3..3ab39e277 100644 --- a/docs/tethys_sdk/jobs/condor_job_type.rst +++ b/docs/tethys_sdk/jobs/condor_job_type.rst @@ -4,6 +4,18 @@ Condor Job Type **Last Updated:** January 2022 +.. important:: + + This feature requires the `condorpy` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `condorpy` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge condorpy + + # pip + pip install condorpy + The :doc:`condor_job_type` (and :doc:`condor_workflow_type`) are used to create jobs to be run by a pool of cluster resources managed by HTCondor. HTCondor makes it possible for jobs to be offloaded from the main web server to a scalable computing cluster, which in turn enables very large scale jobs to be processed. diff --git a/docs/tethys_sdk/jobs/condor_workflow_type.rst b/docs/tethys_sdk/jobs/condor_workflow_type.rst index e950c1d5d..34212c47b 100644 --- a/docs/tethys_sdk/jobs/condor_workflow_type.rst +++ b/docs/tethys_sdk/jobs/condor_workflow_type.rst @@ -4,6 +4,18 @@ Condor Workflow Job Type **Last Updated:** January 2022 +.. important:: + + This feature requires the `condorpy` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `condorpy` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge condorpy + + # pip + pip install condorpy + A Condor Workflow provides a way to run a group of jobs (which can have hierarchical relationships) as a single (Tethys) job. The hierarchical relationships are defined as parent-child relationships. For example, suppose a workflow is defined with three jobs: ``JobA``, ``JobB``, and ``JobC``, which must be run in that order. These jobs would be defined with the following relationships: ``JobA`` is the parent of ``JobB``, and ``JobB`` is the parent of ``JobC``. .. seealso:: diff --git a/docs/tethys_sdk/jobs/dask_job_type.rst b/docs/tethys_sdk/jobs/dask_job_type.rst index e0544c299..bb1a09469 100644 --- a/docs/tethys_sdk/jobs/dask_job_type.rst +++ b/docs/tethys_sdk/jobs/dask_job_type.rst @@ -4,6 +4,18 @@ Dask Job Type **Last Updated:** January 2022 +.. important:: + + This feature requires the `dask` and `tethys_dask_scheduler` libraries to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install these libraries using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge dask tethys_dask_scheduler + + # pip + pip install dask tethys_dask_scheduler + A Dask Job Type wraps Dask functionality in a Tethys Jobs interface. The Tethys Dask Job type supports two different Dask APIs for creating Dask Tasks: ``dask.delayed`` and ``dask.distributed``. Dask Delayed diff --git a/docs/tethys_sdk/layouts/map_layout.rst b/docs/tethys_sdk/layouts/map_layout.rst index a0e09e784..a1a0da9d6 100644 --- a/docs/tethys_sdk/layouts/map_layout.rst +++ b/docs/tethys_sdk/layouts/map_layout.rst @@ -862,6 +862,11 @@ build_param_string .. automethod:: tethys_layouts.views.map_layout.MapLayout.build_param_string +convert_geojson_to_shapefile +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automethod:: tethys_layouts.views.map_layout.MapLayout.convert_geojson_to_shapefile + JavaScript API Documentation ============================ diff --git a/docs/tethys_sdk/rest_api.rst b/docs/tethys_sdk/rest_api.rst index 5ced46055..7cd54adbf 100644 --- a/docs/tethys_sdk/rest_api.rst +++ b/docs/tethys_sdk/rest_api.rst @@ -4,40 +4,50 @@ REST API ******** -REST API's in Tethys Platform use token authentication -(see: http://www.django-rest-framework.org/api-guide/authentication/#tokenauthentication). +.. important:: -You can find the API token for your user on the user management page -(http://[HOST_Portal]/user/[username]/). + This feature requires the `djangorestframework` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `djangorestframework` using conda or pip as follows: -Example Url Map (app.py):: + .. code-block:: bash - UrlMap(name='api_get_data', - url='[your_app_name]/api/get_data', - controller='[your_app_name].api.get_data') + # conda: conda-forge channel strongly recommended + conda install -c conda-forge djangorestframework + # pip + pip install djangorestframework -Example API Controller (api.py):: + **Don't Forget**: If you end up using this feature in your app, add `djangorestframework` as a requirement to your `install.yml`. + +REST API's in Tethys Platform use token authentication (see: http://www.django-rest-framework.org/api-guide/authentication/#tokenauthentication). + +You can find the API token for your user on the user management page (http://[HOST_Portal]/user/[username]/). + +Example API Controller (api.py): + +.. code-block:: python from django.http import JsonResponse from rest_framework.authentication import TokenAuthentication from rest_framework.decorators import api_view, authentication_classes + from tethys_sdk.routing import controller - @api_view(['GET']) + @controller(url='api/get-data') + @api_view(['GET', 'POST']) @authentication_classes((TokenAuthentication,)) - def get_data(request): - ''' - API Controller for getting data - ''' - name = request.GET.get('name') - data = {"name": name} - return JsonResponse(data) + def get_time_series(request): + """ + Controller for the get-time-series REST endpoint. + """ + name = request.GET.get('name', None) + response_data = {'name': name} + return JsonResponse(response_data) +Example Accessing Data: -Example Accessing Data:: +.. code-block:: python >>> import requests - >>> res = requests.get('http://[HOST_Portal]/apps/[your_app_name]/api/get_data?name=oscar', + >>> res = requests.get('http://[HOST_Portal]/apps/[your_app_name]/api/get-data?name=oscar', headers={'Authorization': 'Token asdfqwer1234'}) - >>> da.text - '{"name": "oscar"}' + >>> da.text + '{"name": "oscar"}' diff --git a/docs/tethys_sdk/templating.rst b/docs/tethys_sdk/templating.rst index 5cebc7823..3d754ceab 100644 --- a/docs/tethys_sdk/templating.rst +++ b/docs/tethys_sdk/templating.rst @@ -91,6 +91,17 @@ Examples: See the `Django Tag Reference `_ for a complete list of tags that Django provides. +Tethys Tags ++++++++++++ + +In addition to Django's library of template tags, Tethys also defines a few additional template tags that can be used in your templates. + +.. automodule:: tethys_apps.templatetags.humanize + :members: human_duration + +.. automodule:: tethys_apps.templatetags.app_theme + :members: lighten + Template Inheritance -------------------- diff --git a/docs/tethys_sdk/testing.rst b/docs/tethys_sdk/testing.rst index b29319b7e..c90d5f6fe 100644 --- a/docs/tethys_sdk/testing.rst +++ b/docs/tethys_sdk/testing.rst @@ -49,6 +49,18 @@ https://docs.python.org/2.7/library/unittest.html#module-unittest Testing Controllers that Use OAuth2 Authentication ++++++++++++++++++++++++++++++++++++++++++++++++++ +.. important:: + + This feature requires the `social-auth-app-django` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `social-auth-app-django` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge social-auth-app-django + + # pip + pip install social-auth-app-django + Using the ``force_login`` method above works great for testing controllers where login is required. However, additional steps are required to test controllers that must be authenticated with a specific OAuth2 provider (i.e. specify the ``ensure_oauth_provider`` argument to the ``controller`` decorator). For example, if you have a controller like this: .. code-block:: python diff --git a/docs/tethys_sdk/tethys_services/dataset_services.rst b/docs/tethys_sdk/tethys_services/dataset_services.rst index dda23082a..2b3eec80b 100644 --- a/docs/tethys_sdk/tethys_services/dataset_services.rst +++ b/docs/tethys_sdk/tethys_services/dataset_services.rst @@ -4,6 +4,18 @@ Dataset Services API **Last Updated**: May 2017 +.. important:: + + This feature requires the `tethys_dataset_services` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `tethys_dataset_services` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge tethys_dataset_services + + # pip + pip install tethys_dataset_services + :term:`Dataset services` are web services external to Tethys Platform that can be used to store and publish file-based :term:`datasets` (e.g.: text files, Excel files, zip archives, other model files). Tethys app developers can use the Dataset Services API to access :term:`datasets` for use in their apps and publish any resulting :term:`datasets` their apps may produce. Supported options include `CKAN `_ and `HydroShare `_. Key Concepts @@ -119,13 +131,17 @@ After dataset services have been properly configured, you can use the services t 1. Get a Dataset Service Engine ------------------------------- -Call the ``get_dataset_service()`` method of the app class to get a ``DatasetEngine``:: +Call the ``get_dataset_service()`` method of the app class to get a ``DatasetEngine``: + +.. code-block:: python from my_first_app.app import MyFirstApp as app ckan_engine = app.get_dataset_service('primary_ckan', as_engine=True) -You can also create a ``DatasetEngine`` object directly. This can be useful if you want to vary the credentials for dataset access frequently (e.g.: using user specific credentials):: +You can also create a ``DatasetEngine`` object directly. This can be useful if you want to vary the credentials for dataset access frequently (e.g.: using user specific credentials): + +.. code-block:: python from tethys_dataset_services.engines import CkanDatasetEngine @@ -139,7 +155,9 @@ You can also create a ``DatasetEngine`` object directly. This can be useful if y 2. Use the Dataset Service Engine --------------------------------- -After you have a ``DatasetEngine``, simply call the desired method on it. All ``DatasetEngine`` methods return a dictionary with an item named ``'success'`` that contains a boolean. If the operation was successful, the value of ``'success'`` will be ``True``, otherwise it will be ``False``. If the value of ``'success'`` is ``True``, the dictionary will also contain an item named ``'result'`` that will contain the results. If it is ``False``, the dictionary will contain an item named ``'error'`` that will contain information about the error that occurred. This can be used for debugging purposes as illustrated in the following example:: +After you have a ``DatasetEngine``, simply call the desired method on it. All ``DatasetEngine`` methods return a dictionary with an item named ``'success'`` that contains a boolean. If the operation was successful, the value of ``'success'`` will be ``True``, otherwise it will be ``False``. If the value of ``'success'`` is ``True``, the dictionary will also contain an item named ``'result'`` that will contain the results. If it is ``False``, the dictionary will contain an item named ``'error'`` that will contain information about the error that occurred. This can be used for debugging purposes as illustrated in the following example: + +.. code-block:: python from my_first_app.app import MyFirstApp as app @@ -162,7 +180,7 @@ Use the dataset service engines references above for descriptions of the methods The HydroShare dataset engine uses OAuth 2.0 to authenticate and authorize interactions with the HydroShare via the REST API. This requires passing the ``request`` object as one of the arguments in ``get_dataset_engine()`` method call. Also, to ensure the user is connected to HydroShare, app developers must use the ``ensure_oauth2()`` decorator on any controllers that use the HydroShare dataset engine. For example: - :: + .. code-block:: python from tethys_sdk.services import get_dataset_engine, ensure_oauth2 from .app import MyFirstApp as app diff --git a/docs/tethys_sdk/tethys_services/persistent_store.rst b/docs/tethys_sdk/tethys_services/persistent_store.rst index b196fc55e..a3acca163 100644 --- a/docs/tethys_sdk/tethys_services/persistent_store.rst +++ b/docs/tethys_sdk/tethys_services/persistent_store.rst @@ -4,6 +4,18 @@ Persistent Stores API **Last Updated:** May 2017 +.. important:: + + This feature requires the `psycopg2` and `sqlalchmey` libraries to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install these libraries using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge psycopg2 "sqlalchemy<2" + + # pip + pip install psycopg2 "sqlalchemy<2" + The Persistent Store API streamlines the use of SQL databases in Tethys apps. Using this API, you can provision SQL databases for your app. The databases that will be created are `PostgreSQL `_ databases. Currently, no other databases are supported. The process of creating a new persistent database can be summarized in the following steps: diff --git a/docs/tethys_sdk/tethys_services/spatial_dataset_service/geoserver_reference.rst b/docs/tethys_sdk/tethys_services/spatial_dataset_service/geoserver_reference.rst index d18d6b335..7648148df 100644 --- a/docs/tethys_sdk/tethys_services/spatial_dataset_service/geoserver_reference.rst +++ b/docs/tethys_sdk/tethys_services/spatial_dataset_service/geoserver_reference.rst @@ -114,7 +114,7 @@ These links can be passed to a web mapping client like OpenLayers or Google Maps When you are learning how to use the spatial dataset engine methods, run the commands with the debug parameter set to true. This will automatically pretty print the result dictionary to the console so that you can inspect its contents: - :: + .. code-block:: python # Example method with debug option engine.list_layers(debug=True) diff --git a/docs/tethys_sdk/tethys_services/spatial_dataset_service/thredds_reference.rst b/docs/tethys_sdk/tethys_services/spatial_dataset_service/thredds_reference.rst index 2e6d301c6..90aca4069 100644 --- a/docs/tethys_sdk/tethys_services/spatial_dataset_service/thredds_reference.rst +++ b/docs/tethys_sdk/tethys_services/spatial_dataset_service/thredds_reference.rst @@ -6,6 +6,18 @@ THREDDS Engine (Siphon) Reference **Last Updated**: December 2019 +.. important:: + + This feature requires the `siphon` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `siphon` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge siphon + + # pip + pip install siphon + This guide introduces `Siphon `_, the which is used as the engine for the THREDDS spatial dataset service. Siphon is a 3rd-party library developed by Unidata for interacting with data on remote services, currently focused on THREDDS services. Siphon does not implement the ``SpatialDatasetEngine`` pattern. Example Usage diff --git a/docs/tethys_sdk/tethys_services/spatial_dataset_services.rst b/docs/tethys_sdk/tethys_services/spatial_dataset_services.rst index 162b596b6..448a2801b 100644 --- a/docs/tethys_sdk/tethys_services/spatial_dataset_services.rst +++ b/docs/tethys_sdk/tethys_services/spatial_dataset_services.rst @@ -7,6 +7,18 @@ Spatial Dataset Services API **Last Updated:** December 2019 +.. important:: + + This feature requires the `tethys_dataset_services` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `tethys_dataset_services` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge tethys_dataset_services + + # pip + pip install tethys_dataset_services + Spatial dataset services are web services that can be used to store and publish file-based :term:`spatial datasets` (e.g.: Shapefile, GeoTiff, NetCDF). The spatial datasets published using spatial dataset services are made available in a variety of formats, many of which or more web friendly than the native format (e.g.: PNG, JPEG, GeoJSON, OGC Services). One example of a spatial dataset service is `GeoServer `_, which is capable of storing and serving vector and raster datasets in several popular formats including Shapefiles, GeoTiff, ArcGrid and others. GeoServer serves the data in a variety of formats via the `Open Geospatial Consortium (OGC) `_ standards including `Web Feature Service (WFS) `_, `Web Map Service (WMS) `_, and `Web Coverage Service (WCS) `_. @@ -123,13 +135,17 @@ After spatial dataset services have been properly configured, you can use the se 1. Get an Engine for the Spatial Dataset Service ------------------------------------------------ -Call the ``get_spatial_dataset_service()`` method of the app class to get the engine for the Spatial Dataset Service:: +Call the ``get_spatial_dataset_service()`` method of the app class to get the engine for the Spatial Dataset Service: + +.. code-block:: python from my_first_app.app import MyFirstApp as app geoserver_engine = app.get_spatial_dataset_service('primary_geoserver', as_engine=True) -You can also create a ``SpatialDatasetEngine`` object directly. This can be useful if you want to vary the credentials for dataset access frequently (e.g.: using user specific credentials):: +You can also create a ``SpatialDatasetEngine`` object directly. This can be useful if you want to vary the credentials for dataset access frequently (e.g.: using user specific credentials): + +.. code-block:: python from tethys_dataset_services.engines import GeoServerSpatialDatasetEngine diff --git a/docs/tethys_sdk/tethys_services/spatial_persistent_store.rst b/docs/tethys_sdk/tethys_services/spatial_persistent_store.rst index 028682102..730a5715e 100644 --- a/docs/tethys_sdk/tethys_services/spatial_persistent_store.rst +++ b/docs/tethys_sdk/tethys_services/spatial_persistent_store.rst @@ -4,6 +4,18 @@ Spatial Persistent Stores API **Last Updated:** May 2017 +.. important:: + + This feature requires the `psycopg2`, `sqlalchmey`, and `geoalchemy2` libraries to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install these libraries using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge psycopg2 "sqlalchemy<2" geoalchemy2 + + # pip + pip install psycopg2 "sqlalchemy<2" geoalchemy2 + Persistent store databases can support spatial data types. The spatial capabilities are provided by the `PostGIS `_ extension for the `PostgreSQL `_ database. PostGIS extends the column types of PostgreSQL databases by adding ``geometry``, ``geography``, and ``raster`` types. PostGIS also provides hundreds of database functions that can be used to perform spatial operations on data stored in spatial columns. For more information on PostGIS, see ``_. The following article details the the spatial capabilities of persistent stores in Tethys Platform. This article builds on the concepts and ideas introduced in the :doc:`./persistent_store` documentation. Please review it before continuing. diff --git a/docs/tethys_sdk/tethys_services/web_processing_services.rst b/docs/tethys_sdk/tethys_services/web_processing_services.rst index 05eb481a7..0915ef92a 100644 --- a/docs/tethys_sdk/tethys_services/web_processing_services.rst +++ b/docs/tethys_sdk/tethys_services/web_processing_services.rst @@ -9,7 +9,7 @@ Web Processing Services (WPS) are web services that can be used perform geoproce Web Processing Service Settings =============================== -Using web processing services in your app is accomplised by adding the ``web_processing_service_settings()`` method to your :term:`app class`, which is located in your :term:`app configuration file` (:file:`app.py`). This method should return a list or tuple of ``WebProcessingServiceSetting`` objects. For example: +Using web processing services in your app is accomplished by adding the ``web_processing_service_settings()`` method to your :term:`app class`, which is located in your :term:`app configuration file` (:file:`app.py`). This method should return a list or tuple of ``WebProcessingServiceSetting`` objects. For example: :: @@ -87,6 +87,18 @@ The ``WebProcessingServiceSetting`` can be thought of as a socket for a connecti Working with WPS Services in Apps ================================= +.. important:: + + This feature requires the `owslib` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `owslib` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge owslib + + # pip + pip install owslib + The Web Processing Service API is powered by `OWSLib `_, a Python client that can be used to interact with OGC web services. For detailed explanations the WPS client provided by OWSLib, refer to the `OWSLib WPS Documentation `_. This article only provides a basic introduction to working with the OWSLib WPS client. Get a WPS Engine @@ -130,34 +142,3 @@ After you have retrieved a valid ``owslib.wps.WebProcessingService`` engine obje It is also possible to perform requests using data that are hosted on WFS servers, such as the GeoServer that is provided as part of the Tethys Platform software suite. See the `OWSLib WPS Documentation `_ for more details on how this is to be done. - -Web Processing Service Developer Tool -===================================== - -Tethys Platform provides a developer tool that can be used to browse the sitewide WPS services and the processes that they provide. This tool is useful for formulating new process requests. To use the tool: - -1. Browse to the Developer Tools page of your Tethys Platform by selecting the "Developer" link from the menu at the top of the page. - -2. Select the tool titled "Web Processing Services". - - .. figure:: ../../images/wps_tool/developer_tools_wps.png - :width: 600px - :align: center - -3. Select a WPS service from the list of services that are linked with your Tethys Instance. If no WPS services are linked to your Tethys instance, follow the steps in Sitewide Configuration, above, to setup a WPS service. - - .. figure:: ../../images/wps_tool/wps_tool_services.png - :width: 600px - :align: center - -4. Select the process you wish to view. - - .. figure:: ../../images/wps_tool/wps_tool_processes.png - :width: 600px - :align: center - -A description of the process and the inputs and outputs will be displayed. - - .. figure:: ../../images/wps_tool/wps_tool_buffer.png - :width: 600px - :align: center \ No newline at end of file diff --git a/docs/tutorials/google_earth_engine/part_1/new_app_project.rst b/docs/tutorials/google_earth_engine/part_1/new_app_project.rst index e8dfc87c8..15edef36a 100644 --- a/docs/tutorials/google_earth_engine/part_1/new_app_project.rst +++ b/docs/tutorials/google_earth_engine/part_1/new_app_project.rst @@ -56,17 +56,17 @@ App dependencies should be managed using the :file:`install.yml` instead of the name: earth_engine requirements: - # Putting in a skip true param will skip the entire section. Ignoring the option will assume it be set to False - skip: false - conda: - channels: - - conda-forge - packages: - - earthengine-api - - oauth2client - pip: - - npm: + # Putting in a skip true param will skip the entire section. Ignoring the option will assume it be set to False + skip: false + conda: + channels: + - conda-forge + packages: + - earthengine-api + - oauth2client + pip: + + npm: post: diff --git a/docs/tutorials/google_earth_engine/part_2/file_upload.rst b/docs/tutorials/google_earth_engine/part_2/file_upload.rst index f9ec615b6..9d1f32461 100644 --- a/docs/tutorials/google_earth_engine/part_2/file_upload.rst +++ b/docs/tutorials/google_earth_engine/part_2/file_upload.rst @@ -395,12 +395,16 @@ Now that the file is written to disk, use the built-in ``zipfile`` module to ver In this step you will add the logic to validate that the file contained in the ZIP archive is a shapefile. You will use the ``pyshp`` library to do this, which will introduce a new dependency for the app. -1. Install ``pyshp`` library into your Tethys conda environment. Run the following command in the terminal with your Tethys environment activated: +1. Install ``pyshp`` library into your Tethys conda environment using conda or pip. Run the following command in the terminal with your Tethys environment activated: .. code-block:: bash + # conda: conda-forge channel highly recommended conda install -c conda-forge pyshp + # pip + pip install pyshp + 2. Add ``pyshp`` as a new dependency in the ``install.yml``: .. code-block:: yaml diff --git a/docs/tutorials/google_earth_engine/part_2/rest_api.rst b/docs/tutorials/google_earth_engine/part_2/rest_api.rst index 288896242..4658ff874 100644 --- a/docs/tutorials/google_earth_engine/part_2/rest_api.rst +++ b/docs/tutorials/google_earth_engine/part_2/rest_api.rst @@ -27,7 +27,52 @@ If you wish to use the previous solution as a starting point: cd tethysapp-earth_engine git checkout -b clip-by-asset-solution clip-by-asset-solution-|version| -1. Reorganize Controller Functions into Separate Files +1. Install dependencies +======================= + +The REST API capability requires ``djangorestframework`` to be installed. Install it using conda or pip as follows: + +.. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge djangorestframework + + # pip + pip install djangorestframework + +2. Add dependencies to install.yml +================================== + +Add ``djangorestframework`` to the ``install.yml`` file to ensure it is installed when your app is installed as follows: + +.. code-block:: yaml + + # This file should be committed to your app code. + version: 1.0 + # This should be greater or equal to your tethys-platform in your environment + tethys_version: ">=4.0.0" + # This should match the app - package name in your setup.py + name: earth_engine + + requirements: + # Putting in a skip true param will skip the entire section. Ignoring the option will assume it be set to False + skip: false + conda: + channels: + - conda-forge + packages: + - earthengine-api + - oauth2client + - geojson + - pyshp + - djangorestframework + pip: + + npm: + + post: + +3. Reorganize Controller Functions into Separate Files ====================================================== The :file:`controllers.py` file is beginning to get quite long. To make the controller code more manageable, in this step you will refactor the controllers into several files. @@ -132,7 +177,7 @@ The :file:`controllers.py` file is beginning to get quite long. To make the cont 6. Navigate to ``_ and verify that the app functions as it did before the change. -2. Create New Controller for REST API Endpoint +4. Create New Controller for REST API Endpoint ============================================== REST endpoints are similar to normal controllers. The primary difference is that they typically return data using JSON or XML format instead of HTML. In this step you will create a new controller function for the REST endpoint. @@ -168,7 +213,7 @@ REST endpoints are similar to normal controllers. The primary difference is that 3. Navigate to ``_. You should see an API page that is auto generated by the `Django REST Framework `_ titled **Get Time Series**. The page should display an *HTTP 401 Unauthorized* error and display a result object with detail "Authentication credentials were not provided." -3. Test with Postman Application +5. Test with Postman Application ================================ Most web browsers are surprisingly limited when it comes to testing REST APIs. The reason the test in the previous step resulted in a *401 Unauthorized* is because we sent a request without an authentication token. To more easily test this, you'll want to get a REST client that will allow you to set request headers and parameters. In this tutorial you will use the Postman client to test the REST API as you develop it. @@ -191,7 +236,7 @@ Most web browsers are surprisingly limited when it comes to testing REST APIs. T 9. Press the **Send** button. You should see the same response object as before with the "Authentication credentials were not provided." message. -4. Add Token Authorization Headers to Postman Request +6. Add Token Authorization Headers to Postman Request ===================================================== In this step you will retrieve the API token for your user account and set authentication headers on the request. @@ -210,7 +255,7 @@ In this step you will retrieve the API token for your user account and set authe 7. Press the **Save** button to save your changes to the Postman request. -5. Define Parameters for REST API +7. Define Parameters for REST API ================================= In this step you'll define the parameters that the REST endpoint will accept. If you think of the REST endpoint as a function, then the parameters are like the arguments to the function. The controller will be configured to work with both the ``GET`` and ``POST`` methods for illustration purposes. @@ -292,7 +337,7 @@ In this step you'll define the parameters that the REST endpoint will accept. If 6. Press the **Save** button to save your changes to the Postman request. -6. Validate Platform, Sensor, Product, and Index +8. Validate Platform, Sensor, Product, and Index ================================================ In this step you'll add the validation logic for the ``platform``, ``sensor``, ``product``, and ``index`` parameters. The REST endpoint is like a function shared publicly on the internet--anyone can call it with whatever parameters they want. This includes bots that may try to exploit your website through its REST endpoints. Be sure to only allow valid values through and provide helpful feedback for users of the REST API. @@ -396,7 +441,7 @@ In this step you'll add the validation logic for the ``platform``, ``sensor``, ` 9. Repeat this process, adding first the ``sensor`` parameter, then the ``product`` parameter to confirm that the validation logic is working as expected. -7. Validate Dates +9. Validate Dates ================= In this step you'll add the validation logic for the ``start_date`` and ``end_date`` parameters. There is logic that already exists in the ``viewer`` controller that you can use to validate the date parameters in our REST API function. However, you should avoid copying code to prevent duplicating bugs and make the app easier to maintain. Instead, you will generalize the bit of code from the ``viewer`` controller into a helper function and then use that function in both the ``viewer`` controller and the ``get_time_series`` controller. @@ -581,8 +626,8 @@ In this step you'll add the validation logic for the ``start_date`` and ``end_da * ``start_date`` and ``end_date`` outside of valid range of selected product (see :file:`gee/products.py`) * Incorrect date format given for either date parameter -8. Validate Reducer, Orient, and Scale -====================================== +10. Validate Reducer, Orient, and Scale +======================================= In this step you'll add the validation logic for the ``reducer``, ``orient``, and ``scale`` parameters. The ``reducer`` and ``orient`` parameters each have a short list of valid options and the ``scale`` parameter needs to be a number. @@ -648,8 +693,8 @@ In this step you'll add the validation logic for the ``reducer``, ``orient``, an 8. Change ``scale`` to a valid value other than the default (e.g.: ``150``). Verify this value is returned. -9. Validate Geometry -==================== +11. Validate Geometry +===================== In this step you'll add the logic to validate the ``geometry`` parameter, which should be valid GeoJSON. An optimistic strategy will be used in which an attempt will be made to convert the string into a GeoJSON object. If it fails, then the given string is not valid GeoJSON and an error will be returned. @@ -703,7 +748,7 @@ In this step you'll add the logic to validate the ``geometry`` parameter, which When pasting the ``geometry`` value from above, ensure that there are no new lines / returns after (i.e. press Backspace after pasting). -10. Reuse Existing Helper Function to Get Time Series +12. Reuse Existing Helper Function to Get Time Series ===================================================== With the parameters properly vetted, you are now ready to call the ``get_time_series_from_image_collection`` function. It should be a fairly straightforward call of the function, mapping the REST parameters to the arguments of the function. You will need to make a few minor changes to the function, however, to accommodate the new ``orient`` option. @@ -832,7 +877,7 @@ With the parameters properly vetted, you are now ready to call the ``get_time_se 4. Press the **Send** button to submit the request and verify that the time series is included in the response object. -11. Test & Verify +13. Test & Verify ================= 1. Use Postman to try different values for each of the parameters. Use some that are valid and others that are not to ensure the validation is working. @@ -941,7 +986,7 @@ With the parameters properly vetted, you are now ready to call the ``get_time_se } } -12. Solution +14. Solution ============ This concludes this portion of the GEE Tutorial. You can view the solution on GitHub at ``_ or clone it as follows: diff --git a/docs/tutorials/map_layout/data_prep.rst b/docs/tutorials/map_layout/data_prep.rst index 9ac157d2f..3d0a9b1e8 100644 --- a/docs/tutorials/map_layout/data_prep.rst +++ b/docs/tutorials/map_layout/data_prep.rst @@ -73,7 +73,7 @@ This file will define the conda environment required to run the reproject script 4. Paste the following into the file: -.. code-block:: yml +.. code-block:: yaml name: reproject channels: diff --git a/docs/tutorials/map_layout/new_app_project.rst b/docs/tutorials/map_layout/new_app_project.rst index 6033d174b..9b1179758 100644 --- a/docs/tutorials/map_layout/new_app_project.rst +++ b/docs/tutorials/map_layout/new_app_project.rst @@ -55,17 +55,17 @@ App dependencies should be managed using the :file:`install.yml` instead of the name: map_layout_tutorial requirements: - # Putting in a skip true param will skip the entire section. Ignoring the option will assume it be set to False - skip: false - conda: - channels: - - conda-forge - packages: - - pandas + # Putting in a skip true param will skip the entire section. Ignoring the option will assume it be set to False + skip: false + conda: + channels: + - conda-forge + packages: + - pandas - pip: + pip: - npm: + npm: post: diff --git a/docs/tutorials/thredds/plot_at_location.rst b/docs/tutorials/thredds/plot_at_location.rst index d765d013c..542d84696 100644 --- a/docs/tutorials/thredds/plot_at_location.rst +++ b/docs/tutorials/thredds/plot_at_location.rst @@ -2,7 +2,7 @@ Plot Time Series at Location **************************** -**Last Updated:** June 2022 +**Last Updated:** August 2023 In this tutorial you will add a tool for querying the active THREDDS dataset for time series data at a location and display it on a plot. Topics covered in this tutorial include: @@ -121,7 +121,29 @@ In this step you'll learn to use another Leaflet plugin: `Leaflet.Draw `_ in a web browser and login if necessary. A single tool for drawing markers/points should appear near the top left-hand corner of the map, just below the zoom controls. -2. Create New Plot Controller +2. Install Plotly +================= + +In this step you will create a new controller that will query the dataset at the given location using the NCSS service and then build a plotly plot with the results. + +1. The Plotly View gizmo requires the `plotly` Python package. Install `plotly` as follows running the following command in the terminal: + +.. code-block:: + + # with conda + conda install plotly + + # with pip + pip install plotly + +2. The app now depends on `plotly`, so add it to the `install.yml` file: + +.. code-block:: yaml + + dependencies: + - plotly + +3. Create New Plot Controller ============================= In this step you will create a new controller that will query the dataset at the given location using the NCSS service and then build a plotly plot with the results. @@ -386,7 +408,7 @@ In this step you will create a new controller that will query the dataset at the {% endif %} -3. Load Plot Using JQuery Load +4. Load Plot Using JQuery Load ============================== The `JQuery.load() `_ method is used to call a URL and load the returned HTML into the target element. In this step, you'll use ``jQuery.load()`` to call the ``get-time-series-plot`` endpoint and load the markup for the plot that is returned into a modal for display to the user. This pattern allows you to render the plot dynamically with minimal JavaScript, because the plot is parameterized using Python on the server. @@ -644,7 +666,7 @@ The `JQuery.load() `_ method is used to call a URL update_legend(); }; -4. Test and Verify +5. Test and Verify ================== Browse to ``_ in a web browser and login if necessary. Verify the following: @@ -655,7 +677,7 @@ Browse to ``_ in a web browser and 4. Verify that the plot dialog appears automatically after dropping the marker with the loading image showing. 5. Verify that the plot appears after the data has been queried. -5. Solution +6. Solution =========== This concludes the New App Project portion of the THREDDS Tutorial. You can view the solution on GitHub at ``_ or clone it as follows: diff --git a/environment.yml b/environment.yml index df603b59d..d3cd461c5 100644 --- a/environment.yml +++ b/environment.yml @@ -1,92 +1,114 @@ # environment.yml # Configuration file for creating a Conda Environment with dependencies needed for Tethys Platform. -# Create the environment by running the following command (after installing Miniconda): -# $ conda env create -f environment.yml +# Create the environment by running the following command (after installing Mambaforge): +# $ mamba env create -f environment.yml +# OR +# Create the environment with conda by running the following command +# (after installing Miniconda or similar and conda-libmamba-solver): +# $ conda env create --solver libmamba -f environment.yml name: tethys channels: - conda-forge - - tethysplatform - - defaults dependencies: - python - # encryption dependencies - - bcrypt + # system dependencies + - pyopenssl + - openssl + + # core dependencies + - django=3.2.* + - channels=3.* + - daphne=3.* + - setuptools_scm + - pip + - requests # required by lots of things + - bcrypt # also required by channels, docker, daphne, condorpy + + # Gen CLI commands + - pyyaml + - jinja2 + + # django plugin dependencies + - django-bootstrap5 + - django-model-utils + - django-guardian - # spatial dependencies +###################################### +# Optional Dependencies +###################################### + + # Security Plugins + - django-cors-headers # enable cors? + - django-session-security # session timeouts + - django-axes # tracked failed login attempts + + # Login/Account Plugins + - django-gravatar2 + - django-simple-captcha + - django-mfa2 + - django-recaptcha2 + - social-auth-app-django + - django-oauth-toolkit + + # Misc Plugins + - django-termsandconditions # require users to accept terms and conditions + - django-analytical # track usage analytics + - django-json-widget # enable json widget for app settings + - djangorestframework # enable REST API framework + + # Map Layout - PyShp - # system dependencies - - pyopenssl + # Docker CLI - docker-py - - distro # database dependencies - postgresql - - psycopg2 - - sqlalchemy=1.* # TODO: what will it take to support sqlalchemy 2.0? - - geoalchemy2 + - psycopg2 # required by tethys_dataset_services + - sqlalchemy=1.* # TODO: what will it take to support sqlalchemy 2.0? + - geoalchemy2 # requires sqlalchemy - # plotting dependencies + # plotting Gizmo dependencies - plotly - bokeh<3 - # external services dependencies - - tethys_dataset_services>=2.0.0 - - hs_restclient - - owslib - - requests + # TethysJob Types - dask - - tethys_dask_scheduler>=1.0.2 - - service_identity - condorpy - - siphon - - python-jose - - pyjwt + - tethys_dask_scheduler>=1.0.2 - # datetime dependencies + # external services dependencies + - tethys_dataset_services>=2.0.0 # used with all data services + - hs_restclient # Used with HydroShare + - owslib # used for creating WPS services + - siphon # used with Threads + - python-jose # required by django-mfa2 - used for onelogin backend + + # datetime dependencies for "humanize" template filter - arrow - isodate - # django/plugin dependencies - - django=3.2.* - - channels=3.* - - daphne=3.* - - django-analytical - - django-axes - - django-filter - - djangorestframework - - django-bootstrap5 - - django-cors-headers - - django-model-utils - - django-guardian - - django-gravatar2 - - django-json-widget - - django-mfa2 - - django-recaptcha2 - - django-simple-captcha - - django-session-security - - django-termsandconditions - - social-auth-app-django + # Conda to allow Python API access to Conda Install + - conda + - conda-libmamba-solver + + # Docs + - git - # tests dependencies + # tests/style dependencies - selenium - coverage - factory_boy - - # for now - - pillow - - pip - - future - flake8 - flake8-bugbear - - git - - setuptools_scm - - openssl - # Conda to allow Python API access to Conda Install - - conda - - conda-libmamba-solver + + # not needed: +# - future # required by django-json-widget (not used directly) +# - service_identity # required by channels, daphne (not used directly) +# - django-filter # I'm not sure where this was being used +# - pyjwt # required by social-auth-app-django ( not used directly ) diff --git a/min_env.yml b/min_env.yml new file mode 100644 index 000000000..550239ff0 --- /dev/null +++ b/min_env.yml @@ -0,0 +1,38 @@ +# environment.yml +# Configuration file for creating a Conda Environment with dependencies needed for Tethys Platform. +# Create the environment by running the following command (after installing Mambaforge): +# $ mamba env create -f min_env.yml +# OR +# Create the environment with conda by running the following command (after installing Miniconda or similar): +# $ conda env create -f min_env.yml + +name: tethys + +channels: + - conda-forge + +dependencies: + - python + + # system dependencies + - pyopenssl + - openssl + + # core dependencies + - django=3.2.* + - channels=3.* + - daphne=3.* + - setuptools_scm + - pip + - requests # required by lots of things + - bcrypt # also required by channels, docker, daphne, condorpy + + # Gen CLI commands + - pyyaml + - jinja2 + + + # django plugin dependencies + - django-bootstrap5 + - django-model-utils + - django-guardian diff --git a/tests/unit_tests/test_tethys_cli/test_docker_commands.py b/tests/unit_tests/test_tethys_cli/test_docker_commands.py index 84c736434..2c841f9fb 100644 --- a/tests/unit_tests/test_tethys_cli/test_docker_commands.py +++ b/tests/unit_tests/test_tethys_cli/test_docker_commands.py @@ -1,4 +1,3 @@ -import importlib import unittest from unittest import mock import tethys_cli.docker_commands as cli_docker_commands @@ -38,10 +37,6 @@ def make_args( ) return args - def test_curses_import_error(self): - with mock.patch.dict("sys.modules", {"curses": None}): - importlib.reload(cli_docker_commands) - def test_get_docker_client(self): dc = cli_docker_commands.ContainerMetadata.get_docker_client() self.assertIs(dc, self.mock_dc) @@ -1256,14 +1251,14 @@ def test_uih_get_valid_directory_input_oserror( @mock.patch("tethys_cli.docker_commands.write_pretty_output") @mock.patch("tethys_cli.docker_commands.curses") - @mock.patch("tethys_cli.docker_commands.platform.system") + @mock.patch("tethys_cli.docker_commands.has_module") def test_log_pull_stream_linux_with_id_bad_status( - self, mock_platform_system, mock_curses, mock_pretty_output + self, mock_has_module, mock_curses, mock_pretty_output ): mock_stream = [ b'{ "id":"358464", "status":"foo", "progress":"bar" }' ] - mock_platform_system.return_value = "Linux" + mock_has_module.return_value = True mock_curses.initscr().getmaxyx.return_value = 1, 80 cli_docker_commands.log_pull_stream(mock_stream) @@ -1293,14 +1288,14 @@ def test_log_pull_stream_linux_with_id_bad_status( @mock.patch("tethys_cli.docker_commands.write_pretty_output") @mock.patch("tethys_cli.docker_commands.curses") - @mock.patch("tethys_cli.docker_commands.platform.system") + @mock.patch("tethys_cli.docker_commands.has_module") def test_log_pull_stream_linux_with_id_progress_status( - self, mock_platform_system, mock_curses, mock_pretty_output + self, mock_has_module, mock_curses, mock_pretty_output ): mock_stream = [ b'{ "id":"358464", "status":"Downloading", "progress":"bar" }' ] - mock_platform_system.return_value = "Linux" + mock_has_module.return_value = True mock_curses.initscr().getmaxyx.return_value = 1, 80 cli_docker_commands.log_pull_stream(mock_stream) @@ -1324,15 +1319,15 @@ def test_log_pull_stream_linux_with_id_progress_status( @mock.patch("tethys_cli.docker_commands.write_pretty_output") @mock.patch("tethys_cli.docker_commands.curses") - @mock.patch("tethys_cli.docker_commands.platform.system") + @mock.patch("tethys_cli.docker_commands.has_module") def test_log_pull_stream_linux_with_id_status( - self, mock_platform_system, mock_curses, mock_pretty_output + self, mock_has_module, mock_curses, mock_pretty_output ): mock_stream = [ b'{ "id":"358464", "status":"Downloading", "progress":"bar" }\r\n' b'{ "id":"358464", "status":"Pulling fs layer", "progress":"baz" }' ] - mock_platform_system.return_value = "Linux" + mock_has_module.return_value = True mock_curses.initscr().getmaxyx.return_value = 1, 80 cli_docker_commands.log_pull_stream(mock_stream) @@ -1356,12 +1351,12 @@ def test_log_pull_stream_linux_with_id_status( @mock.patch("tethys_cli.docker_commands.write_pretty_output") @mock.patch("tethys_cli.docker_commands.curses") - @mock.patch("tethys_cli.docker_commands.platform.system") + @mock.patch("tethys_cli.docker_commands.has_module") def test_log_pull_stream_linux_with_no_id( - self, mock_platform_system, mock_curses, mock_pretty_output + self, mock_has_module, mock_curses, mock_pretty_output ): mock_stream = [b'{ "status":"foo", "progress":"bar" }'] - mock_platform_system.return_value = "Linux" + mock_has_module.return_value = True mock_curses.initscr().getmaxyx.return_value = 1, 80 cli_docker_commands.log_pull_stream(mock_stream) @@ -1380,16 +1375,16 @@ def test_log_pull_stream_linux_with_no_id( @mock.patch("tethys_cli.docker_commands.write_pretty_output") @mock.patch("tethys_cli.docker_commands.curses") - @mock.patch("tethys_cli.docker_commands.platform.system") + @mock.patch("tethys_cli.docker_commands.has_module") def test_log_pull_stream_linux_with_curses_error( - self, mock_platform_system, mock_curses, mock_pretty_output + self, mock_has_module, mock_curses, mock_pretty_output ): import curses mock_stream = [ b'{ "id":"358464", "status":"Downloading", "progress":"bar" }\r\n' ] - mock_platform_system.return_value = "Linux" + mock_has_module.return_value = True mock_curses.initscr().getmaxyx.return_value = 1, 80 mock_curses.error = ( curses.error @@ -1416,12 +1411,12 @@ def test_log_pull_stream_linux_with_curses_error( self.assertEqual("", po_call_args[0][0][0]) @mock.patch("tethys_cli.docker_commands.write_pretty_output") - @mock.patch("tethys_cli.docker_commands.platform.system") - def test_log_pull_stream_windows(self, mock_platform_system, mock_pretty_output): + @mock.patch("tethys_cli.docker_commands.has_module") + def test_log_pull_stream_windows(self, mock_has_module, mock_pretty_output): mock_stream = [ b'{ "id":"358464", "status":"Downloading", "progress":"bar" }' ] - mock_platform_system.return_value = "Windows" + mock_has_module.return_value = False cli_docker_commands.log_pull_stream(mock_stream) diff --git a/tests/unit_tests/test_tethys_cli/test_gen_commands.py b/tests/unit_tests/test_tethys_cli/test_gen_commands.py index 5b7f8a3f2..ab1ebe2d9 100644 --- a/tests/unit_tests/test_tethys_cli/test_gen_commands.py +++ b/tests/unit_tests/test_tethys_cli/test_gen_commands.py @@ -2,7 +2,6 @@ from unittest import mock from pathlib import Path -import tethys_cli.gen_commands as tethys_gen_commands from tethys_cli.gen_commands import ( get_environment_value, get_settings_value, @@ -39,26 +38,6 @@ def setUp(self): def tearDown(self): pass - @mock.patch("tethys_cli.cli_colors.write_warning") - def test_no_conda(self, mock_warn): - import tethys_cli.gen_commands as tethys_gen_commands - from importlib import reload - import builtins - - real_import = builtins.__import__ - - def mock_import(name, *args): - if name == "conda.cli.python_api": - raise ModuleNotFoundError - else: - return real_import(name, *args) - - builtins.__import__ = mock_import - reload(tethys_gen_commands) - builtins.__import__ = real_import - self.assertEqual(tethys_gen_commands.has_conda, False) - mock_warn.assert_called_once() - def test_get_environment_value(self): result = get_environment_value(value_name="DJANGO_SETTINGS_MODULE") @@ -191,94 +170,16 @@ def test_generate_command_portal_yaml__tethys_home_not_exists( @mock.patch("tethys_cli.gen_commands.write_info") @mock.patch("tethys_cli.gen_commands.render_template") - @mock.patch("tethys_cli.gen_commands.linux_distribution") - @mock.patch("tethys_cli.gen_commands.os.path.exists") - @mock.patch("tethys_cli.gen_commands.get_environment_value") - @mock.patch("tethys_cli.gen_commands.open", new_callable=mock.mock_open) - @mock.patch("tethys_cli.gen_commands.os.path.isfile") - def test_generate_command_asgi_service_option_nginx_conf_redhat( - self, - mock_os_path_isfile, - mock_file, - mock_env, - mock_os_path_exists, - mock_linux_distribution, - mock_render_template, - mock_write_info, - ): - mock_args = mock.MagicMock(conda_prefix=False) - mock_args.type = GEN_ASGI_SERVICE_OPTION - mock_args.directory = None - mock_os_path_isfile.return_value = False - mock_env.side_effect = ["/foo/conda", "conda_env"] - mock_os_path_exists.return_value = True - mock_linux_distribution.return_value = ["redhat"] - mock_file.return_value = mock.mock_open(read_data="user foo_user").return_value - - generate_command(args=mock_args) - - mock_os_path_isfile.assert_called_once() - mock_file.assert_called() - mock_env.assert_called_with("CONDA_PREFIX") - mock_os_path_exists.assert_any_call("/etc/nginx/nginx.conf") - context = mock_render_template.call_args.args[1] - self.assertEqual("http-", context["user_option_prefix"]) - self.assertEqual("foo_user", context["nginx_user"]) - - mock_write_info.assert_called() - - @mock.patch("tethys_cli.gen_commands.write_info") - @mock.patch("tethys_cli.gen_commands.render_template") - @mock.patch("tethys_cli.gen_commands.linux_distribution") - @mock.patch("tethys_cli.gen_commands.os.path.exists") - @mock.patch("tethys_cli.gen_commands.get_environment_value") - @mock.patch("tethys_cli.gen_commands.open", new_callable=mock.mock_open) - @mock.patch("tethys_cli.gen_commands.os.path.isfile") - def test_generate_command_asgi_service_option_nginx_conf_ubuntu( - self, - mock_os_path_isfile, - mock_file, - mock_env, - mock_os_path_exists, - mock_linux_distribution, - mock_render_template, - mock_write_info, - ): - mock_args = mock.MagicMock(conda_prefix=False) - mock_args.type = GEN_ASGI_SERVICE_OPTION - mock_args.directory = None - mock_os_path_isfile.return_value = False - mock_env.side_effect = ["/foo/conda", "conda_env"] - mock_os_path_exists.return_value = True - mock_linux_distribution.return_value = "ubuntu" - mock_file.return_value = mock.mock_open(read_data="user foo_user").return_value - - generate_command(args=mock_args) - - mock_os_path_isfile.assert_called_once() - mock_file.assert_called() - mock_env.assert_called_with("CONDA_PREFIX") - mock_os_path_exists.assert_any_call("/etc/nginx/nginx.conf") - context = mock_render_template.call_args.args[1] - self.assertEqual("", context["user_option_prefix"]) - self.assertEqual("foo_user", context["nginx_user"]) - - mock_write_info.assert_called() - - @mock.patch("tethys_cli.gen_commands.write_info") - @mock.patch("tethys_cli.gen_commands.render_template") - @mock.patch("tethys_cli.gen_commands.linux_distribution") @mock.patch("tethys_cli.gen_commands.os.path.exists") @mock.patch("tethys_cli.gen_commands.get_environment_value") @mock.patch("tethys_cli.gen_commands.open", new_callable=mock.mock_open) @mock.patch("tethys_cli.gen_commands.os.path.isfile") - def test_generate_command_asgi_service_option_nginx_conf_not_linux( + def test_generate_command_asgi_service_option_nginx_conf( self, mock_os_path_isfile, mock_file, mock_env, mock_os_path_exists, - mock_linux_distribution, mock_render_template, mock_write_info, ): @@ -288,7 +189,6 @@ def test_generate_command_asgi_service_option_nginx_conf_not_linux( mock_os_path_isfile.return_value = False mock_env.side_effect = ["/foo/conda", "conda_env"] mock_os_path_exists.return_value = True - mock_linux_distribution.side_effect = Exception mock_file.return_value = mock.mock_open(read_data="user foo_user").return_value generate_command(args=mock_args) @@ -298,7 +198,6 @@ def test_generate_command_asgi_service_option_nginx_conf_not_linux( mock_env.assert_called_with("CONDA_PREFIX") mock_os_path_exists.assert_any_call("/etc/nginx/nginx.conf") context = mock_render_template.call_args.args[1] - self.assertEqual("", context["user_option_prefix"]) self.assertEqual("foo_user", context["nginx_user"]) mock_write_info.assert_called() @@ -325,7 +224,6 @@ def test_generate_command_asgi_service_option( mock_write_info.assert_called() @mock.patch("tethys_cli.gen_commands.write_info") - @mock.patch("tethys_cli.gen_commands.linux_distribution") @mock.patch("tethys_cli.gen_commands.get_environment_value") @mock.patch("tethys_cli.gen_commands.open", new_callable=mock.mock_open) @mock.patch("tethys_cli.gen_commands.os.path.isfile") @@ -334,7 +232,6 @@ def test_generate_command_asgi_service_option_distro( mock_os_path_isfile, mock_file, mock_env, - mock_distribution, mock_write_info, ): mock_args = mock.MagicMock(conda_prefix=False) @@ -342,7 +239,6 @@ def test_generate_command_asgi_service_option_distro( mock_args.directory = None mock_os_path_isfile.return_value = False mock_env.side_effect = ["/foo/conda", "conda_env"] - mock_distribution.return_value = ("redhat", "linux", "") generate_command(args=mock_args) @@ -540,7 +436,7 @@ def test_generate_requirements_option( @mock.patch("tethys_cli.gen_commands.os.path.join") @mock.patch("tethys_cli.gen_commands.write_info") @mock.patch("tethys_cli.gen_commands.Template") - @mock.patch("tethys_cli.gen_commands.safe_load") + @mock.patch("tethys_cli.gen_commands.yaml.safe_load") @mock.patch("tethys_cli.gen_commands.run_command") @mock.patch("tethys_cli.gen_commands.open", new_callable=mock.mock_open) @mock.patch("tethys_cli.gen_commands.os.path.isfile") @@ -591,7 +487,7 @@ def test_generate_command_metayaml( @mock.patch("tethys_cli.gen_commands.write_info") @mock.patch("tethys_cli.gen_commands.derive_version_from_conda_environment") - @mock.patch("tethys_cli.gen_commands.safe_load") + @mock.patch("tethys_cli.gen_commands.yaml.safe_load") @mock.patch("tethys_cli.gen_commands.open", new_callable=mock.mock_open) def test_gen_meta_yaml_overriding_dependencies( self, _, mock_load, mock_dvfce, mock_write_info @@ -883,14 +779,14 @@ def test_download_vendor_static_files_no_npm(self, mock_call, mock_error): mock_call.assert_called_once() mock_error.assert_called_once() - @mock.patch.object(tethys_gen_commands, "has_conda") + @mock.patch("tethys_cli.gen_commands.has_module") @mock.patch("tethys_cli.gen_commands.write_error") @mock.patch("tethys_cli.gen_commands.call") def test_download_vendor_static_files_no_npm_no_conda( - self, mock_call, mock_error, mock_has_conda + self, mock_call, mock_error, mock_has_module ): mock_call.side_effect = FileNotFoundError - mock_has_conda.__bool__ = lambda self: False + mock_has_module.return_value = False download_vendor_static_files(mock.MagicMock()) mock_call.assert_called_once() mock_error.assert_called_once() diff --git a/tests/unit_tests/test_tethys_cli/test_install_commands.py b/tests/unit_tests/test_tethys_cli/test_install_commands.py index c8b033366..aedcb9d1e 100644 --- a/tests/unit_tests/test_tethys_cli/test_install_commands.py +++ b/tests/unit_tests/test_tethys_cli/test_install_commands.py @@ -22,25 +22,6 @@ def setUp(self): ) self.app.save() - @mock.patch("tethys_cli.cli_colors.write_warning") - def test_no_conda(self, mock_warn): - from importlib import reload - import builtins - - real_import = builtins.__import__ - - def mock_import(name, *args): - if name == "conda.cli.python_api": - raise ModuleNotFoundError - else: - return real_import(name, *args) - - builtins.__import__ = mock_import - reload(install_commands) - builtins.__import__ = real_import - self.assertEqual(install_commands.has_conda, False) - mock_warn.assert_called_once() - @mock.patch("tethys_cli.install_commands.exit") @mock.patch("tethys_cli.cli_colors.pretty_output") def test_open_file_error(self, mock_pretty_output, mock_exit): @@ -1005,13 +986,13 @@ def test_conda_and_pip_package_install( mock_exit.assert_called_with(0) @mock.patch("tethys_cli.install_commands.write_warning") - @mock.patch.object(install_commands, "has_conda") + @mock.patch("tethys_cli.install_commands.has_module") @mock.patch("tethys_cli.install_commands.run_services") @mock.patch("tethys_cli.install_commands.call") @mock.patch("tethys_cli.install_commands.exit") @mock.patch("tethys_cli.cli_colors.pretty_output") def test_conda_install_no_conda( - self, mock_pretty_output, mock_exit, mock_call, _, mock_has_conda, mock_warn + self, mock_pretty_output, mock_exit, mock_call, _, mock_has_module, mock_warn ): file_path = self.root_app_path / "install-dep.yml" args = mock.MagicMock( @@ -1025,7 +1006,7 @@ def test_conda_install_no_conda( without_dependencies=False, ) mock_exit.side_effect = SystemExit - mock_has_conda.__bool__ = lambda self: False + mock_has_module.return_value = False self.assertRaises(SystemExit, install_commands.install_command, args) @@ -1048,13 +1029,13 @@ def test_conda_install_no_conda( mock_exit.assert_called_with(0) @mock.patch("tethys_cli.install_commands.write_warning") - @mock.patch.object(install_commands, "has_conda") + @mock.patch("tethys_cli.install_commands.has_module") @mock.patch("tethys_cli.install_commands.run_services") @mock.patch("tethys_cli.install_commands.call") @mock.patch("tethys_cli.install_commands.exit") @mock.patch("tethys_cli.cli_colors.pretty_output") def test_conda_install_no_conda_error( - self, mock_pretty_output, mock_exit, mock_call, _, mock_has_conda, mock_warn + self, mock_pretty_output, mock_exit, mock_call, _, mock_has_module, mock_warn ): file_path = self.root_app_path / "install-dep.yml" args = mock.MagicMock( @@ -1068,7 +1049,7 @@ def test_conda_install_no_conda_error( without_dependencies=False, ) mock_exit.side_effect = SystemExit - mock_has_conda.__bool__ = lambda self: False + mock_has_module.return_value = False mock_call.side_effect = [Exception, None, None, None] self.assertRaises(SystemExit, install_commands.install_command, args) diff --git a/tests/unit_tests/test_tethys_portal/test_optional_dependencies.py b/tests/unit_tests/test_tethys_portal/test_optional_dependencies.py new file mode 100644 index 000000000..02b61405a --- /dev/null +++ b/tests/unit_tests/test_tethys_portal/test_optional_dependencies.py @@ -0,0 +1,41 @@ +import unittest +from tethys_portal import optional_dependencies + + +class TestStaticDependency(unittest.TestCase): + def setUp(self): + self.module = "test_module" + self.import_error = "error" + self.failed_import = optional_dependencies.FailedImport( + self.module, self.import_error + ) + + def tearDown(self): + pass + + def test_failed_import_init(self): + module = "test_module" + import_error = "error" + failed_import = optional_dependencies.FailedImport(module, import_error) + self.assertEqual(failed_import.module_name, module) + self.assertEqual(failed_import.error, import_error) + + def test_failed_import_call(self): + self.assertRaises(ImportError, lambda: self.failed_import.test()) + + def test_failed_import_getattr(self): + self.assertRaises(ImportError, lambda: self.failed_import.test) + + def test_failed_import_getitem(self): + self.assertRaises(ImportError, lambda: self.failed_import["test"]) + + def test__attempt_import_error(self): + module = optional_dependencies._attempt_import( + self.module, from_module=None, error_message=None + ) + self.assertIsInstance(module, optional_dependencies.FailedImport) + + def test_verify_import(self): + self.assertRaises( + ImportError, optional_dependencies.verify_import, self.failed_import + ) diff --git a/tethys_apps/admin.py b/tethys_apps/admin.py index 8bdf107e2..b48d828c3 100644 --- a/tethys_apps/admin.py +++ b/tethys_apps/admin.py @@ -18,12 +18,10 @@ from django.utils.html import format_html from django.shortcuts import reverse from django.db import models -from django_json_widget.widgets import JSONEditorWidget from tethys_quotas.admin import TethysAppQuotasSettingInline, UserQuotasSettingInline from guardian.admin import GuardedModelAdmin from guardian.shortcuts import assign_perm, remove_perm from guardian.models import GroupObjectPermission -from mfa.models import User_Keys from tethys_quotas.utilities import get_quota, _convert_storage_units from tethys_quotas.handlers.workspace import WorkspaceQuotaHandler from tethys_apps.models import ( @@ -40,7 +38,17 @@ PersistentStoreDatabaseSetting, ProxyApp, ) +from tethys_portal.optional_dependencies import ( + optional_import, + has_module, + MissingOptionalDependency, +) +# optional imports +User_Keys = optional_import("User_Keys", from_module="mfa.models") +JSONEditorWidget = optional_import( + "JSONEditorWidget", from_module="django_json_widget.widgets" +) tethys_log = logging.getLogger("tethys." + __name__) @@ -95,13 +103,14 @@ class JSONCustomSettingInline(TethysAppSettingInline): width_default = "100%" height_default = "300px" - formfield_overrides = { - models.JSONField: { - "widget": JSONEditorWidget( - width=width_default, height=height_default, options=options_default - ) - }, - } + if has_module(JSONEditorWidget): + formfield_overrides = { + models.JSONField: { + "widget": JSONEditorWidget( + width=width_default, height=height_default, options=options_default + ) + }, + } class DatasetServiceSettingInline(TethysAppSettingInline): @@ -529,7 +538,7 @@ def register_user_keys_admin(): User_Keys._meta.verbose_name = "Users MFA Key" User_Keys._meta.verbose_name_plural = "Users MFA Keys" admin.site.register(User_Keys, UserKeyAdmin) - except ProgrammingError: + except (ProgrammingError, MissingOptionalDependency): tethys_log.warning("Unable to register UserKeys.") @@ -540,7 +549,8 @@ class ProxyAppAdmin(GuardedModelAdmin): register_custom_group() admin.site.unregister(User) admin.site.register(User, CustomUser) -register_user_keys_admin() +if has_module(User_Keys): + register_user_keys_admin() admin.site.register(ProxyApp, ProxyAppAdmin) admin.site.register(TethysApp, TethysAppAdmin) admin.site.register(TethysExtension, TethysExtensionAdmin) diff --git a/tethys_apps/base/bokeh_handler.py b/tethys_apps/base/bokeh_handler.py index 9c94da996..cbe0c4433 100644 --- a/tethys_apps/base/bokeh_handler.py +++ b/tethys_apps/base/bokeh_handler.py @@ -10,8 +10,6 @@ from functools import wraps # Third Party Imports -from bokeh.document import Document -from bokeh.embed import server_document # Django Imports from django.http.request import HttpRequest @@ -20,6 +18,12 @@ # Tethys Imports from tethys_sdk.workspaces import get_user_workspace, get_app_workspace +from tethys_portal.optional_dependencies import optional_import + +# Optional Imports +Document = optional_import("Document", from_module="bokeh.document") +server_document = optional_import("server_document", from_module="bokeh.embed") + def with_request(handler): @wraps(handler) diff --git a/tethys_apps/migrations/0001_initial_40.py b/tethys_apps/migrations/0001_initial_40.py deleted file mode 100644 index 170d77487..000000000 --- a/tethys_apps/migrations/0001_initial_40.py +++ /dev/null @@ -1,390 +0,0 @@ -# Generated by Django 3.2.12 on 2022-08-12 16:39 -# flake8: noqa - -import django.contrib.postgres.fields -from django.db import migrations, models -import django.db.models.deletion -import tethys_apps.base.mixins -import tethys_services.models - - -class Migration(migrations.Migration): - initial = True - - dependencies = [ - ("tethys_services", "0001_initial_40"), - ("tethys_compute", "0001_initial_40"), - ] - - operations = [ - migrations.CreateModel( - name="TethysApp", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("package", models.CharField(default="", max_length=200, unique=True)), - ("name", models.CharField(default="", max_length=200)), - ( - "description", - models.TextField(blank=True, default="", max_length=1000), - ), - ("enable_feedback", models.BooleanField(default=False)), - ( - "feedback_emails", - django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - blank=True, max_length=200, null=True - ), - default=list, - size=None, - ), - ), - ("tags", models.CharField(blank=True, default="", max_length=200)), - ("index", models.CharField(default="", max_length=200)), - ("icon", models.CharField(default="", max_length=200)), - ("root_url", models.CharField(default="", max_length=200)), - ("color", models.CharField(default="", max_length=10)), - ("enabled", models.BooleanField(default=True)), - ("show_in_apps_library", models.BooleanField(default=True)), - ("order", models.IntegerField(default=0)), - ], - options={ - "verbose_name": "Tethys App", - "verbose_name_plural": "Installed Apps", - }, - bases=(models.Model, tethys_apps.base.mixins.TethysBaseMixin), - ), - migrations.CreateModel( - name="TethysAppSetting", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("name", models.CharField(default="", max_length=200)), - ( - "description", - models.TextField(blank=True, default="", max_length=1000), - ), - ("required", models.BooleanField(default=True)), - ("initializer", models.CharField(default="", max_length=1000)), - ("initialized", models.BooleanField(default=False)), - ( - "tethys_app", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="settings_set", - to="tethys_apps.tethysapp", - ), - ), - ], - ), - migrations.CreateModel( - name="WebProcessingServiceSetting", - fields=[ - ( - "tethysappsetting_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_apps.tethysappsetting", - ), - ), - ( - "web_processing_service", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="tethys_services.webprocessingservice", - ), - ), - ], - bases=("tethys_apps.tethysappsetting",), - ), - migrations.CreateModel( - name="SpatialDatasetServiceSetting", - fields=[ - ( - "tethysappsetting_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_apps.tethysappsetting", - ), - ), - ( - "engine", - models.CharField( - choices=[ - ( - "tethys_dataset_services.engines.GeoServerSpatialDatasetEngine", - "GeoServer", - ), - ("thredds-engine", "THREDDS"), - ], - default="tethys_dataset_services.engines.GeoServerSpatialDatasetEngine", - max_length=200, - ), - ), - ( - "spatial_dataset_service", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="tethys_services.spatialdatasetservice", - ), - ), - ], - bases=("tethys_apps.tethysappsetting",), - ), - migrations.CreateModel( - name="PersistentStoreDatabaseSetting", - fields=[ - ( - "tethysappsetting_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_apps.tethysappsetting", - ), - ), - ("spatial", models.BooleanField(default=False)), - ("dynamic", models.BooleanField(default=False)), - ( - "persistent_store_service", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="tethys_services.persistentstoreservice", - ), - ), - ], - bases=("tethys_apps.tethysappsetting",), - ), - migrations.CreateModel( - name="PersistentStoreConnectionSetting", - fields=[ - ( - "tethysappsetting_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_apps.tethysappsetting", - ), - ), - ( - "persistent_store_service", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="tethys_services.persistentstoreservice", - ), - ), - ], - bases=("tethys_apps.tethysappsetting",), - ), - migrations.CreateModel( - name="DatasetServiceSetting", - fields=[ - ( - "tethysappsetting_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_apps.tethysappsetting", - ), - ), - ( - "engine", - models.CharField( - choices=[ - ( - "tethys_dataset_services.engines.CkanDatasetEngine", - "CKAN", - ), - ( - "tethys_dataset_services.engines.HydroShareDatasetEngine", - "HydroShare", - ), - ], - default="tethys_dataset_services.engines.CkanDatasetEngine", - max_length=200, - ), - ), - ( - "dataset_service", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="tethys_services.datasetservice", - ), - ), - ], - bases=("tethys_apps.tethysappsetting",), - ), - migrations.CreateModel( - name="CustomSetting", - fields=[ - ( - "tethysappsetting_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_apps.tethysappsetting", - ), - ), - ("value", models.CharField(blank=True, default="", max_length=1024)), - ("default", models.CharField(blank=True, default="", max_length=1024)), - ( - "type", - models.CharField( - choices=[ - ("STRING", "String"), - ("INTEGER", "Integer"), - ("FLOAT", "Float"), - ("BOOLEAN", "Boolean"), - ("UUID", "UUID"), - ], - default="STRING", - max_length=200, - ), - ), - ], - bases=("tethys_apps.tethysappsetting",), - ), - migrations.CreateModel( - name="TethysExtension", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("package", models.CharField(default="", max_length=200, unique=True)), - ("name", models.CharField(default="", max_length=200)), - ( - "description", - models.TextField(blank=True, default="", max_length=1000), - ), - ("root_url", models.CharField(default="", max_length=200)), - ("enabled", models.BooleanField(default=True)), - ], - options={ - "verbose_name": "Tethys Extension", - "verbose_name_plural": "Installed Extensions", - }, - bases=(models.Model, tethys_apps.base.mixins.TethysBaseMixin), - ), - migrations.CreateModel( - name="SchedulerSetting", - fields=[ - ( - "tethysappsetting_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_apps.tethysappsetting", - ), - ), - ( - "engine", - models.CharField( - choices=[("htcondor", "HTCondor"), ("dask", "Dask")], - default="htcondor", - max_length=200, - ), - ), - ( - "scheduler_service", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.CASCADE, - to="tethys_compute.scheduler", - ), - ), - ], - bases=("tethys_apps.tethysappsetting",), - ), - migrations.CreateModel( - name="ProxyApp", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("name", models.CharField(max_length=100, unique=True)), - ( - "endpoint", - models.CharField( - max_length=1024, - validators=[tethys_services.models.validate_url], - ), - ), - ( - "logo_url", - models.CharField( - blank=True, - max_length=100, - validators=[tethys_services.models.validate_url], - ), - ), - ("description", models.TextField(blank=True, max_length=2048)), - ("tags", models.CharField(blank=True, default="", max_length=200)), - ("enabled", models.BooleanField(default=True)), - ("show_in_apps_library", models.BooleanField(default=True)), - ("order", models.IntegerField(default=0)), - ], - options={ - "verbose_name": "Proxy App", - "verbose_name_plural": "Proxy Apps", - }, - ), - ] diff --git a/tethys_apps/migrations/0001_initial_41.py b/tethys_apps/migrations/0001_initial_41.py index d49d0db82..aa5ab0d7d 100644 --- a/tethys_apps/migrations/0001_initial_41.py +++ b/tethys_apps/migrations/0001_initial_41.py @@ -6,11 +6,6 @@ class Migration(migrations.Migration): - replaces = [ - ("tethys_apps", "0001_initial_40"), - ("tethys_apps", "0002_auto_20221130_2305"), - ] - initial = True dependencies = [ diff --git a/tethys_apps/migrations/0002_auto_20221130_2305.py b/tethys_apps/migrations/0002_auto_20221130_2305.py deleted file mode 100644 index a4c997e0d..000000000 --- a/tethys_apps/migrations/0002_auto_20221130_2305.py +++ /dev/null @@ -1,32 +0,0 @@ -# Generated by Django 3.2.16 on 2022-11-30 23:05 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("tethys_apps", "0001_initial_40"), - ] - - operations = [ - migrations.AddField( - model_name="proxyapp", - name="back_url", - field=models.URLField(blank=True, max_length=512), - ), - migrations.AddField( - model_name="proxyapp", - name="open_in_new_tab", - field=models.BooleanField(default=True), - ), - migrations.AlterField( - model_name="proxyapp", - name="endpoint", - field=models.URLField(max_length=512), - ), - migrations.AlterField( - model_name="proxyapp", - name="logo_url", - field=models.URLField(blank=True, max_length=512), - ), - ] diff --git a/tethys_apps/models.py b/tethys_apps/models.py index 961b46e8f..99c73afc4 100644 --- a/tethys_apps/models.py +++ b/tethys_apps/models.py @@ -7,7 +7,6 @@ * License: BSD 2-Clause ******************************************************************************** """ -import sqlalchemy from django.dispatch import receiver import logging import uuid @@ -22,7 +21,6 @@ PersistentStorePermissionError, PersistentStoreInitializerError, ) -from sqlalchemy.orm import sessionmaker from tethys_apps.base.mixins import TethysBaseMixin from tethys_compute.models.condor.condor_scheduler import CondorScheduler from tethys_compute.models.dask.dask_scheduler import DaskScheduler @@ -30,6 +28,11 @@ from tethys_sdk.testing import is_testing_environment, get_test_db_name from tethys_apps.base.function_extractor import TethysFunctionExtractor from tethys_apps.utilities import secrets_signed_unsigned_value +from tethys_portal.optional_dependencies import optional_import, has_module + +# optional imports +sqlalchemy = optional_import("sqlalchemy") +sessionmaker = optional_import("sessionmaker", from_module="sqlalchemy.orm") log = logging.getLogger("tethys") @@ -203,7 +206,8 @@ class TethysAppSetting(models.Model): DB Model for Tethys App Settings """ - objects = InheritanceManager() + if has_module(InheritanceManager): + objects = InheritanceManager() tethys_app = models.ForeignKey( TethysApp, on_delete=models.CASCADE, related_name="settings_set" diff --git a/tethys_apps/templates/tethys_apps/app_base.html b/tethys_apps/templates/tethys_apps/app_base.html index 3c2b1ce4a..7b6c1d76f 100644 --- a/tethys_apps/templates/tethys_apps/app_base.html +++ b/tethys_apps/templates/tethys_apps/app_base.html @@ -1,4 +1,4 @@ -{% load static app_theme tethys_gizmos terms_tags analytical %} +{% load static app_theme tethys_gizmos %} {# Allows custom attributes to be added to the html tag #} @@ -11,7 +11,9 @@ {# Allows custom attributes to be added to the head tag #} - {% analytical_head_top %} + {% if has_analytical %} + {% include "analytical_head_top.html" %} + {% endif %} {% comment "meta explanation" %} Add custom meta tags to the page. Call block.super to get the default tags @@ -135,17 +137,23 @@ {% gizmo_dependencies global_js %} {% endblock %} + {% if has_session_security %} {% block session_timeout_modal %} {% include 'session_security/all.html' %} {% endblock %} + {% endif %} - {% analytical_head_bottom %} + {% if has_analytical %} + {% include "analytical_head_bottom.html" %} + {% endif %} {# Allows custom attributes to be added to the body tag #} - {% analytical_body_top %} + {% if has_analytical %} + {% include "analytical_body_top.html" %} + {% endif %} {% block app_content_wrapper_override %}
@@ -273,7 +281,9 @@ {% endblock %} {% block terms-of-service-override %} - {% show_terms_if_not_agreed %} + {% if has_terms %} + {% include "terms.html" %} + {% endif %} {% endblock %} {% block page_attributes_override %} @@ -310,6 +320,8 @@ {% gizmo_dependencies js %} {% endblock %} - {% analytical_body_bottom %} + {% if has_analytical %} + {% include "analytical_body_bottom.html" %} + {% endif %} diff --git a/tethys_apps/templatetags/app_theme.py b/tethys_apps/templatetags/app_theme.py index 2f8ccb3df..2300ab930 100644 --- a/tethys_apps/templatetags/app_theme.py +++ b/tethys_apps/templatetags/app_theme.py @@ -8,13 +8,23 @@ @register.filter def lighten(hex_color, percentage): + """ + Lighten a hex color by a certain percentage and return the lightened color. + + Args: + hex_color: A hex color value in the format "#2d3436". + percentage: A number (0-100) representing a percentage to lighten the color by. + + Returns: A hex color value in the format "#2d3436" + + """ if not re.search(hex_regex_pattern, hex_color): raise ValueError( f'Given "{hex_color}", but needs to be in hex color format (e.g.: "#2d3436").' ) # Extract the hex strings for the RGB components - rgb_hex = [hex_color[x : x + 2] for x in [1, 3, 5]] + rgb_hex = [hex_color[x:x + 2] for x in [1, 3, 5]] # Convert RGB hex strings to integer numbers rgb_int = [int(x, 16) for x in rgb_hex] diff --git a/tethys_apps/templatetags/humanize.py b/tethys_apps/templatetags/humanize.py index 2fbfaf832..c1f0640bf 100644 --- a/tethys_apps/templatetags/humanize.py +++ b/tethys_apps/templatetags/humanize.py @@ -1,7 +1,12 @@ -import arrow -import isodate from django import template +from tethys_portal.optional_dependencies import optional_import + +# optional imports +arrow = optional_import("arrow") +isodate = optional_import("isodate") + + register = template.Library() @@ -10,11 +15,29 @@ def human_duration(iso_duration_str): """ Converts an ISO 8601 formatted duration to a humanized time from now (UTC). + .. important:: + + This feature requires the `arrow` and `isodate` libraries to be installed. Starting with Tethys 5.0 or if you are + using `microtethys`, you will need to install these libraries using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge arrow isodate + + # pip + pip install arrow isodate + Args: iso_duration_str: An ISO 8601 formatted string (e.g. "P1DT3H6M") Returns: str: humanized string representing the amount of time from now (e.g.: "in 30 minutes"). + + Usage: + {% load static humanize %} + + {{ P1DT3H6M|human_duration }} """ time_change = isodate.parse_duration(iso_duration_str) now = arrow.utcnow() diff --git a/tethys_apps/utilities.py b/tethys_apps/utilities.py index 7b04f8ffb..dc3c14c27 100644 --- a/tethys_apps/utilities.py +++ b/tethys_apps/utilities.py @@ -10,15 +10,17 @@ import importlib import logging import os +from pathlib import Path + import pkgutil import yaml -from pathlib import Path + from django.core.signing import Signer from django.core import signing -from tethys_apps.exceptions import TethysAppSettingNotAssigned - from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.utils._os import safe_join + +from tethys_apps.exceptions import TethysAppSettingNotAssigned from .harvester import SingletonHarvester tethys_log = logging.getLogger("tethys." + __name__) diff --git a/tethys_cli/cli_helpers.py b/tethys_cli/cli_helpers.py index ec0ba650e..346e535d7 100644 --- a/tethys_cli/cli_helpers.py +++ b/tethys_cli/cli_helpers.py @@ -1,11 +1,11 @@ import os import sys import subprocess -import bcrypt -import yaml +from pathlib import Path +import bcrypt import django -from pathlib import Path +import yaml from tethys_apps.base.testing.environment import set_testing_environment from tethys_apps.utilities import ( diff --git a/tethys_cli/db_commands.py b/tethys_cli/db_commands.py index 23f1276e8..58ddf93f6 100644 --- a/tethys_cli/db_commands.py +++ b/tethys_cli/db_commands.py @@ -472,6 +472,10 @@ def configure_tethys_db(**kwargs): _prompt_if_error(start_db_server, **kwargs) if "postgresql" in kwargs.get("db_engine"): _prompt_if_error(create_tethys_db, **kwargs) + if "sqlite" in kwargs.get("db_engine"): + # Make sure the parent directory for the database exists + db_path = Path(kwargs["db_name"]) + db_path.parent.mkdir(exist_ok=True, parents=True) migrate_tethys_db(**kwargs) create_portal_superuser(**kwargs) diff --git a/tethys_cli/docker_commands.py b/tethys_cli/docker_commands.py index 5f0606f04..c78aa5b9e 100644 --- a/tethys_cli/docker_commands.py +++ b/tethys_cli/docker_commands.py @@ -7,21 +7,20 @@ * License: BSD 2-Clause ******************************************************************************** """ -try: - import curses -except Exception: # pragma: no cover - pass # curses not available on Windows -import platform import os import json from abc import ABC, abstractmethod import getpass -import docker -from docker.types import Mount -from docker.errors import NotFound as DockerNotFound from tethys_cli.cli_colors import write_pretty_output, write_error, write_warning from tethys_apps.utilities import get_tethys_home_dir +from tethys_portal.optional_dependencies import optional_import, has_module + +# optional imports +curses = optional_import("curses") # curses not available on Windows +docker = optional_import("docker") +Mount = optional_import("Mount", from_module="docker.types") +DockerNotFound = optional_import("NotFound", from_module="docker.errors") __all__ = [ @@ -1008,7 +1007,7 @@ def log_pull_stream(stream): """ Handle the printing of pull statuses """ - if platform.system() == "Windows": # i.e. can't uses curses + if not has_module(curses): for block in stream: lines = [line for line in block.split(b"\r\n") if line] for line in lines: diff --git a/tethys_cli/gen_commands.py b/tethys_cli/gen_commands.py index fb34cf2f8..bfe4d2c7c 100644 --- a/tethys_cli/gen_commands.py +++ b/tethys_cli/gen_commands.py @@ -15,10 +15,8 @@ from pathlib import Path from subprocess import call, run -import yaml -from yaml import safe_load -from distro import linux_distribution from jinja2 import Template +import yaml from django.conf import settings @@ -35,13 +33,12 @@ from .site_commands import SITE_SETTING_CATEGORIES -has_conda = False -try: - from conda.cli.python_api import run_command, Commands +from tethys_portal.optional_dependencies import optional_import, has_module - has_conda = True -except ModuleNotFoundError: - write_warning("Conda not found. Some functionality will not be available.") +# optional imports +run_command, Commands = optional_import( + ("run_command", "Commands"), from_module="conda.cli.python_api" +) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tethys_portal.settings") @@ -291,15 +288,6 @@ def gen_asgi_service(args): conda_home = Path(conda_prefix).parents[1] conda_env_name = Path(conda_prefix).name - user_option_prefix = "" - - try: - linux_distro = linux_distribution(full_distribution_name=0)[0] - if linux_distro in ["redhat", "centos"]: - user_option_prefix = "http-" - except Exception: - pass - context = { "nginx_user": nginx_user, "port": args.tethys_port, @@ -309,7 +297,6 @@ def gen_asgi_service(args): "conda_env_name": conda_env_name, "tethys_src": TETHYS_SRC, "tethys_home": TETHYS_HOME, - "user_option_prefix": user_option_prefix, "is_micromamba": args.micromamba, } return context @@ -439,7 +426,7 @@ def derive_version_from_conda_environment(dep_str, level="none"): def gen_meta_yaml(args): environment_file_path = os.path.join(TETHYS_SRC, "environment.yml") with open(environment_file_path, "r") as env_file: - environment = safe_load(env_file) + environment = yaml.safe_load(env_file) dependencies = environment.get("dependencies", []) run_requirements = [] @@ -480,7 +467,7 @@ def download_vendor_static_files(args, cwd=None): install_instructions = ( "To get npm you must install nodejs. Run the following command to install nodejs:" "\n\n\tconda install -c conda-forge nodejs\n" - if has_conda + if has_module(run_command) else "For help installing npm see: https://docs.npmjs.com/downloading-and-installing-node-js-and-npm" ) msg = ( diff --git a/tethys_cli/install_commands.py b/tethys_cli/install_commands.py index 8b59ac885..9a80dcafc 100644 --- a/tethys_cli/install_commands.py +++ b/tethys_cli/install_commands.py @@ -5,6 +5,8 @@ from pathlib import Path from subprocess import call, Popen, PIPE, STDOUT from argparse import Namespace +from collections.abc import Mapping + from django.core.exceptions import ObjectDoesNotExist, ValidationError from tethys_cli.cli_colors import write_msg, write_error, write_warning, write_success @@ -21,15 +23,12 @@ ) from .gen_commands import download_vendor_static_files -from collections.abc import Mapping +from tethys_portal.optional_dependencies import optional_import, has_module -has_conda = False -try: - from conda.cli.python_api import run_command as conda_run, Commands - - has_conda = True -except ModuleNotFoundError: - write_warning("Conda not found. Some functionality will not be available.") +# optional imports +conda_run, Commands = optional_import( + ("run_command", "Commands"), from_module="conda.cli.python_api" +) FNULL = open(os.devnull, "w") @@ -746,7 +745,7 @@ def install_command(args): write_warning("Skipping package installation.") else: if validate_schema("conda", requirements_config): # noqa: E501 - if has_conda: + if has_module(conda_run): conda_config = requirements_config["conda"] install_packages( conda_config, update_installed=args.update_installed diff --git a/tethys_compute/migrations/0001_initial_40.py b/tethys_compute/migrations/0001_initial_40.py deleted file mode 100644 index d73bbf403..000000000 --- a/tethys_compute/migrations/0001_initial_40.py +++ /dev/null @@ -1,417 +0,0 @@ -# Generated by Django 3.2.12 on 2022-08-12 16:42 - -from django.conf import settings -import django.contrib.postgres.fields -from django.db import migrations, models -import django.db.models.deletion -import tethys_compute.models.dask.dask_field - - -class Migration(migrations.Migration): - initial = True - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name="CondorPyJob", - fields=[ - ("condorpyjob_id", models.AutoField(primary_key=True, serialize=False)), - ("_attributes", models.JSONField(blank=True, default=dict, null=True)), - ("_num_jobs", models.IntegerField(default=1)), - ( - "_remote_input_files", - django.contrib.postgres.fields.ArrayField( - base_field=models.CharField( - blank=True, max_length=1024, null=True - ), - default=list, - size=None, - ), - ), - ], - ), - migrations.CreateModel( - name="CondorPyWorkflow", - fields=[ - ( - "condorpyworkflow_id", - models.AutoField(primary_key=True, serialize=False), - ), - ("_max_jobs", models.JSONField(blank=True, default=dict, null=True)), - ("_config", models.CharField(blank=True, max_length=1024, null=True)), - ], - ), - migrations.CreateModel( - name="CondorWorkflowNode", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("name", models.CharField(max_length=1024)), - ( - "pre_script", - models.CharField(blank=True, max_length=1024, null=True), - ), - ( - "pre_script_args", - models.CharField(blank=True, max_length=1024, null=True), - ), - ( - "post_script", - models.CharField(blank=True, max_length=1024, null=True), - ), - ( - "post_script_args", - models.CharField(blank=True, max_length=1024, null=True), - ), - ("variables", models.JSONField(blank=True, default=dict, null=True)), - ("priority", models.IntegerField(blank=True, null=True)), - ("category", models.CharField(blank=True, max_length=128, null=True)), - ("retry", models.PositiveSmallIntegerField(blank=True, null=True)), - ("retry_unless_exit_value", models.IntegerField(blank=True, null=True)), - ("pre_skip", models.IntegerField(blank=True, null=True)), - ("abort_dag_on", models.IntegerField(blank=True, null=True)), - ( - "abort_dag_on_return_value", - models.IntegerField(blank=True, null=True), - ), - ("dir", models.CharField(blank=True, max_length=1024, null=True)), - ("noop", models.BooleanField(default=False)), - ("done", models.BooleanField(default=False)), - ( - "parent_nodes", - models.ManyToManyField( - related_name="children_nodes", - to="tethys_compute.CondorWorkflowNode", - ), - ), - ( - "workflow", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="node_set", - to="tethys_compute.condorpyworkflow", - ), - ), - ], - ), - migrations.CreateModel( - name="Scheduler", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("name", models.CharField(max_length=1024)), - ("host", models.CharField(max_length=1024)), - ], - options={ - "verbose_name": "Scheduler", - "verbose_name_plural": "Schedulers", - }, - ), - migrations.CreateModel( - name="TethysJob", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("name", models.CharField(max_length=1024)), - ( - "description", - models.CharField(blank=True, default="", max_length=2048), - ), - ("label", models.CharField(max_length=1024)), - ("creation_time", models.DateTimeField(auto_now_add=True)), - ("execute_time", models.DateTimeField(blank=True, null=True)), - ("start_time", models.DateTimeField(blank=True, null=True)), - ("completion_time", models.DateTimeField(blank=True, null=True)), - ("workspace", models.CharField(default="", max_length=1024)), - ( - "extended_properties", - models.JSONField(blank=True, default=dict, null=True), - ), - ( - "_process_results_function", - models.CharField(blank=True, max_length=1024, null=True), - ), - ( - "_status", - models.CharField( - choices=[ - ("PEN", "Pending"), - ("SUB", "Submitted"), - ("RUN", "Running"), - ("VAR", "Various"), - ("PAS", "Paused"), - ("COM", "Complete"), - ("ERR", "Error"), - ("ABT", "Aborted"), - ("VCP", "Various-Complete"), - ("RES", "Results-Ready"), - ("OTH", "Other"), - ], - default="PEN", - max_length=3, - ), - ), - ( - "user", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to=settings.AUTH_USER_MODEL, - ), - ), - ( - "groups", - models.ManyToManyField( - blank=True, - related_name="tethys_jobs", - to="auth.Group", - verbose_name="groups", - ), - ), - ( - "status_message", - models.CharField(blank=True, max_length=2048, null=True), - ), - ], - options={ - "verbose_name": "Job", - }, - ), - migrations.CreateModel( - name="BasicJob", - fields=[ - ( - "tethysjob_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_compute.tethysjob", - ), - ), - ], - bases=("tethys_compute.tethysjob",), - ), - migrations.CreateModel( - name="CondorBase", - fields=[ - ( - "tethysjob_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_compute.tethysjob", - ), - ), - ("cluster_id", models.IntegerField(blank=True, default=0)), - ("remote_id", models.CharField(blank=True, max_length=32, null=True)), - ], - bases=("tethys_compute.tethysjob",), - ), - migrations.CreateModel( - name="CondorScheduler", - fields=[ - ( - "scheduler_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_compute.scheduler", - ), - ), - ("username", models.CharField(blank=True, max_length=1024, null=True)), - ("password", models.CharField(blank=True, max_length=1024, null=True)), - ( - "private_key_path", - models.CharField(blank=True, max_length=1024, null=True), - ), - ( - "private_key_pass", - models.CharField(blank=True, max_length=1024, null=True), - ), - ("port", models.IntegerField(blank=True, default=22, null=True)), - ], - options={ - "verbose_name": "HTCondor Scheduler", - "verbose_name_plural": "HTCondor Schedulers", - }, - bases=("tethys_compute.scheduler",), - ), - migrations.CreateModel( - name="CondorWorkflowJobNode", - fields=[ - ( - "condorpyjob_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - to="tethys_compute.condorpyjob", - ), - ), - ( - "condorworkflownode_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_compute.condorworkflownode", - ), - ), - ], - bases=("tethys_compute.condorworkflownode", "tethys_compute.condorpyjob"), - ), - migrations.CreateModel( - name="DaskScheduler", - fields=[ - ( - "scheduler_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_compute.scheduler", - ), - ), - ("timeout", models.IntegerField(blank=True, default=0)), - ("heartbeat_interval", models.IntegerField(blank=True, default=0)), - ("dashboard", models.CharField(blank=True, max_length=255, null=True)), - ], - options={ - "verbose_name": "Dask Scheduler", - "verbose_name_plural": "Dask Schedulers", - }, - bases=("tethys_compute.scheduler",), - ), - migrations.CreateModel( - name="CondorJob", - fields=[ - ( - "condorpyjob_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - to="tethys_compute.condorpyjob", - ), - ), - ( - "condorbase_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_compute.condorbase", - ), - ), - ], - bases=("tethys_compute.condorbase", "tethys_compute.condorpyjob"), - ), - migrations.CreateModel( - name="CondorWorkflow", - fields=[ - ( - "condorpyworkflow_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - to="tethys_compute.condorpyworkflow", - ), - ), - ( - "condorbase_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_compute.condorbase", - ), - ), - ], - bases=("tethys_compute.condorbase", "tethys_compute.condorpyworkflow"), - ), - migrations.CreateModel( - name="DaskJob", - fields=[ - ( - "tethysjob_ptr", - models.OneToOneField( - auto_created=True, - on_delete=django.db.models.deletion.CASCADE, - parent_link=True, - primary_key=True, - serialize=False, - to="tethys_compute.tethysjob", - ), - ), - ("key", models.CharField(max_length=1024, null=True)), - ("forget", models.BooleanField(default=False)), - ( - "result", - tethys_compute.models.dask.dask_field.DaskSerializedField( - blank=True, null=True - ), - ), - ( - "scheduler", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to="tethys_compute.daskscheduler", - ), - ), - ], - bases=("tethys_compute.tethysjob",), - ), - migrations.AddField( - model_name="condorbase", - name="scheduler", - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to="tethys_compute.condorscheduler", - ), - ), - ] diff --git a/tethys_compute/migrations/0001_initial_41.py b/tethys_compute/migrations/0001_initial_41.py index 504a74116..82c101a4d 100644 --- a/tethys_compute/migrations/0001_initial_41.py +++ b/tethys_compute/migrations/0001_initial_41.py @@ -7,11 +7,6 @@ class Migration(migrations.Migration): - replaces = [ - ("tethys_compute", "0001_initial_40"), - ("tethys_compute", "0002_alter_condorpyjob__remote_input_files"), - ] - initial = True dependencies = [ diff --git a/tethys_compute/migrations/0002_alter_condorpyjob__remote_input_files.py b/tethys_compute/migrations/0002_alter_condorpyjob__remote_input_files.py deleted file mode 100644 index 3b77f7162..000000000 --- a/tethys_compute/migrations/0002_alter_condorpyjob__remote_input_files.py +++ /dev/null @@ -1,49 +0,0 @@ -# Generated by Django 3.2.16 on 2022-12-02 18:07 - -import json -from django.db import migrations, models - -remote_input_files = {} - - -def save_remote_files_as_json(apps, schema_editor): - CondorPyJob = apps.get_model("tethys_compute", "CondorPyJob") - db_alias = schema_editor.connection.alias - for job in CondorPyJob.objects.using(db_alias).all(): - remote_input_files[job.condorpyjob_id] = json.dumps(job._remote_input_files) - - -def load_saved_remote_files(apps, schema_editor): - CondorPyJob = apps.get_model("tethys_compute", "CondorPyJob") - db_alias = schema_editor.connection.alias - for job_id, files in remote_input_files.items(): - job = CondorPyJob.objects.using(db_alias).get(condorpyjob_id=job_id) - job._remote_input_files = files - job.save() - - -def save_remote_files_as_list(apps, schema_editor): - CondorPyJob = apps.get_model("tethys_compute", "CondorPyJob") - db_alias = schema_editor.connection.alias - for job in CondorPyJob.objects.using(db_alias).all(): - remote_input_files[job.id] = json.loads(job._remote_input_files) - - -class Migration(migrations.Migration): - dependencies = [ - ("tethys_compute", "0001_initial_40"), - ] - - operations = [ - migrations.RunPython(save_remote_files_as_json, load_saved_remote_files), - migrations.RemoveField( - model_name="condorpyjob", - name="_remote_input_files", - ), - migrations.AddField( - model_name="condorpyjob", - name="_remote_input_files", - field=models.JSONField(blank=True, default=list, null=True), - ), - migrations.RunPython(load_saved_remote_files, save_remote_files_as_list), - ] diff --git a/tethys_compute/models/condor/condor_py_job.py b/tethys_compute/models/condor/condor_py_job.py index eb5e134e2..6baf2e2e3 100644 --- a/tethys_compute/models/condor/condor_py_job.py +++ b/tethys_compute/models/condor/condor_py_job.py @@ -6,11 +6,15 @@ * Copyright: (c) Aquaveo 2018 ******************************************************************************** """ +from tethys_portal.optional_dependencies import optional_import import os -from condorpy import Templates, Job from django.db import models +# optional imports +Templates = optional_import("Templates", from_module="condorpy") +Job = optional_import("Job", from_module="condorpy") + class CondorPyJob(models.Model): """ diff --git a/tethys_compute/models/condor/condor_py_workflow.py b/tethys_compute/models/condor/condor_py_workflow.py index 202461137..4cd4769c2 100644 --- a/tethys_compute/models/condor/condor_py_workflow.py +++ b/tethys_compute/models/condor/condor_py_workflow.py @@ -6,9 +6,13 @@ * Copyright: (c) Aquaveo 2018 ******************************************************************************** """ -from condorpy import Workflow +from tethys_portal.optional_dependencies import optional_import + from django.db import models +# optional imports +Workflow = optional_import("Workflow", from_module="condorpy") + class CondorPyWorkflow(models.Model): """ diff --git a/tethys_compute/models/condor/condor_workflow_node.py b/tethys_compute/models/condor/condor_workflow_node.py index da19c9060..fef7c190c 100644 --- a/tethys_compute/models/condor/condor_workflow_node.py +++ b/tethys_compute/models/condor/condor_workflow_node.py @@ -8,11 +8,14 @@ """ from abc import abstractmethod -from condorpy import Node from django.db import models from model_utils.managers import InheritanceManager from tethys_compute.models.condor.condor_py_workflow import CondorPyWorkflow +from tethys_portal.optional_dependencies import optional_import + +# optional imports +Node = optional_import("Node", from_module="condorpy") class CondorWorkflowNode(models.Model): diff --git a/tethys_compute/models/dask/dask_field.py b/tethys_compute/models/dask/dask_field.py index 34f65278c..89cdc243c 100644 --- a/tethys_compute/models/dask/dask_field.py +++ b/tethys_compute/models/dask/dask_field.py @@ -1,7 +1,12 @@ from ast import literal_eval as make_tuple from django.db import models from django.core.exceptions import ValidationError -from distributed.protocol.serialize import serialize, deserialize +from tethys_portal.optional_dependencies import optional_import + +# optional imports +serialize, deserialize = optional_import( + ("serialize", "deserialize"), from_module="distributed.protocol.serialize" +) class DaskSerializedField(models.Field): diff --git a/tethys_compute/models/dask/dask_job.py b/tethys_compute/models/dask/dask_job.py index 847be8db8..4f0be22a9 100644 --- a/tethys_compute/models/dask/dask_job.py +++ b/tethys_compute/models/dask/dask_job.py @@ -8,15 +8,21 @@ """ import logging import datetime +import json from django.utils import timezone from django.db import models -from dask.delayed import Delayed -from dask.distributed import Client, Future, fire_and_forget + from tethys_compute.models.tethys_job import TethysJob from tethys_compute.models.dask.dask_scheduler import DaskScheduler from tethys_compute.models.dask.dask_field import DaskSerializedField -import json +from tethys_portal.optional_dependencies import optional_import + +# optional imports +Delayed = optional_import("Delayed", from_module="dask.delayed") +Client = optional_import("Client", from_module="dask.distributed") +Future = optional_import("Future", from_module="dask.distributed") +fire_and_forget = optional_import("fire_and_forget", from_module="dask.distributed") log = logging.getLogger("tethys." + __name__) client_fire_forget = None diff --git a/tethys_compute/models/dask/dask_scheduler.py b/tethys_compute/models/dask/dask_scheduler.py index 9a0295e9d..ea751b6b5 100644 --- a/tethys_compute/models/dask/dask_scheduler.py +++ b/tethys_compute/models/dask/dask_scheduler.py @@ -10,7 +10,10 @@ from django.db import models from tethys_compute.models.scheduler import Scheduler from tethys_compute.models.dask.dask_job_exception import DaskJobException -from dask.distributed import Client +from tethys_portal.optional_dependencies import optional_import + +# optional imports +Client = optional_import("client", from_module="dask.distributed") log = logging.getLogger("tethys." + __name__) diff --git a/tethys_compute/views/dask_dashboard.py b/tethys_compute/views/dask_dashboard.py index 50217f3e9..59e033148 100644 --- a/tethys_compute/views/dask_dashboard.py +++ b/tethys_compute/views/dask_dashboard.py @@ -1,6 +1,9 @@ from django.shortcuts import render, reverse -from bokeh.embed import server_document from tethys_compute.models.dask.dask_scheduler import DaskScheduler +from tethys_portal.optional_dependencies import optional_import + +# optional imports +server_document = optional_import("server_document", from_module="bokeh.embed") def dask_dashboard(request, dask_scheduler_id, page="status"): diff --git a/tethys_config/context_processors.py b/tethys_config/context_processors.py index 7a1745664..5b34caff8 100644 --- a/tethys_config/context_processors.py +++ b/tethys_config/context_processors.py @@ -8,6 +8,7 @@ ******************************************************************************** """ import datetime as dt +from tethys_portal.optional_dependencies import optional_import, has_module def tethys_global_settings_context(request): @@ -15,7 +16,11 @@ def tethys_global_settings_context(request): Add the current Tethys app metadata to the template context. """ from .models import Setting - from termsandconditions.models import TermsAndConditions + + # optional imports + TermsAndConditions = optional_import( + "TermsAndConditions", from_module="termsandconditions.models" + ) # Get settings site_globals = Setting.as_dict() @@ -43,7 +48,8 @@ def tethys_global_settings_context(request): site_globals["secondary_text_hover_color"] = "#aaaaaa" # Get terms and conditions - site_globals.update({"documents": TermsAndConditions.get_active_terms_list()}) + if has_module(TermsAndConditions): + site_globals.update({"documents": TermsAndConditions.get_active_terms_list()}) context = { "site_globals": site_globals, diff --git a/tethys_gizmos/gizmo_options/bokeh_view.py b/tethys_gizmos/gizmo_options/bokeh_view.py index 19a5749a3..27f088481 100644 --- a/tethys_gizmos/gizmo_options/bokeh_view.py +++ b/tethys_gizmos/gizmo_options/bokeh_view.py @@ -1,13 +1,16 @@ # coding=utf-8 -from bokeh.embed import components -from bokeh.resources import Resources -from bokeh.settings import settings as bk_settings - from django.conf import settings from django.utils.functional import classproperty from .base import TethysGizmoOptions +from tethys_portal.optional_dependencies import optional_import + +# optional imports +components = optional_import("components", from_module="bokeh.embed") +Resources = optional_import("Resources", from_module="bokeh.resources") +bk_settings = optional_import("settings", from_module="bokeh.settings") + __all__ = ["BokehView"] diff --git a/tethys_gizmos/gizmo_options/plotly_view.py b/tethys_gizmos/gizmo_options/plotly_view.py index 6d2863ae5..9e3fb77af 100644 --- a/tethys_gizmos/gizmo_options/plotly_view.py +++ b/tethys_gizmos/gizmo_options/plotly_view.py @@ -1,7 +1,9 @@ # coding=utf-8 -import plotly.offline as opy - from .base import TethysGizmoOptions +from tethys_portal.optional_dependencies import optional_import + +# optional imports +opy = optional_import("plotly.offline") __all__ = ["PlotlyView"] diff --git a/tethys_gizmos/templatetags/tethys_gizmos.py b/tethys_gizmos/templatetags/tethys_gizmos.py index f35c35fa8..06eebd732 100644 --- a/tethys_gizmos/templatetags/tethys_gizmos.py +++ b/tethys_gizmos/templatetags/tethys_gizmos.py @@ -19,12 +19,15 @@ from django.templatetags.static import static from django.core.serializers.json import DjangoJSONEncoder -import plotly # noqa: F401 -from plotly.offline.offline import get_plotlyjs from tethys_apps.harvester import SingletonHarvester from ..gizmo_options.base import TethysGizmoOptions import tethys_sdk.gizmos +from tethys_portal.optional_dependencies import optional_import + +# optional imports +plotly = optional_import("plotly") +get_plotlyjs = optional_import("get_plotlyjs", from_module="plotly.offline.offline") GIZMO_NAME_PROPERTY = "gizmo_name" GIZMO_NAME_MAP = {} diff --git a/tethys_gizmos/views/gizmos/jobs_table.py b/tethys_gizmos/views/gizmos/jobs_table.py index 6842e9d36..b01356cec 100644 --- a/tethys_gizmos/views/gizmos/jobs_table.py +++ b/tethys_gizmos/views/gizmos/jobs_table.py @@ -5,8 +5,11 @@ from django.template.loader import render_to_string from tethys_compute.models import TethysJob, CondorWorkflow, DaskJob, DaskScheduler from tethys_gizmos.gizmo_options.jobs_table import JobsTable -from bokeh.embed import server_document from tethys_sdk.gizmos import SelectInput +from tethys_portal.optional_dependencies import optional_import + +# optional imports +server_document = optional_import("server_document", from_module="bokeh.embed") log = logging.getLogger("tethys.tethys_gizmos.views.jobs_table") diff --git a/tethys_layouts/views/map_layout.py b/tethys_layouts/views/map_layout.py index 7fb312dda..3ed9d67b3 100644 --- a/tethys_layouts/views/map_layout.py +++ b/tethys_layouts/views/map_layout.py @@ -6,6 +6,7 @@ * Copyright: (c) Aquaveo 2021 ******************************************************************************** """ +from tethys_portal.optional_dependencies import optional_import from abc import ABCMeta import collections from io import BytesIO @@ -20,7 +21,6 @@ from django.http import HttpResponse, JsonResponse from django.shortcuts import render from django.utils.functional import classproperty -import shapefile # PyShp from tethys_layouts.exceptions import TethysLayoutPropertyException from tethys_layouts.mixins.map_layout import MapLayoutMixin @@ -35,6 +35,9 @@ SelectInput, ) +# optional imports +shapefile = optional_import("shapefile") # PyShp + log = logging.getLogger(f"tethys.{__name__}") @@ -797,6 +800,20 @@ def convert_geojson_to_shapefile(self, request, *args, **kwargs): AJAX handler that converts GeoJSON data into a shapefile for download. Credit to: https://github.com/TipsForGIS/geoJSONToShpFile/blob/master/geoJ.py + .. important:: + + This method requires the `pyshp` library to be installed. Starting with Tethys 5.0 or if you are using `microtethys`, you will need to install `django-json-widget` using conda or pip as follows: + + .. code-block:: bash + + # conda: conda-forge channel strongly recommended + conda install -c conda-forge pyshp + + # pip + pip install pyshp + + **Don't Forget**: If you end up using this method in your app, add `pyshp` as a requirement to your `install.yml`. + Args: request(HttpRequest): The request. diff --git a/tethys_portal/context_processors.py b/tethys_portal/context_processors.py new file mode 100644 index 000000000..531a5ac5c --- /dev/null +++ b/tethys_portal/context_processors.py @@ -0,0 +1,25 @@ +""" +******************************************************************************** +* Name: context_processors.py +* Author: Scott Christensen +* Created On: 2023 +* Copyright: (c) Brigham Young University 2014 +* License: BSD 2-Clause +******************************************************************************** +""" + +from .optional_dependencies import has_module + + +def tethys_portal_context(request): + context = { + "has_analytical": has_module("analytical"), + "has_recaptcha": has_module("snowpenguin.django.recaptcha2"), + "has_terms": has_module("termsandconditions"), + "has_mfa": has_module("mfa"), + "has_gravatar": has_module("django_gravatar"), + "has_session_security": has_module("session_security"), + "has_oauth2_provider": has_module("oauth2_provider"), + } + + return context diff --git a/tethys_portal/middleware.py b/tethys_portal/middleware.py index 0be18ee22..7c0bf5177 100644 --- a/tethys_portal/middleware.py +++ b/tethys_portal/middleware.py @@ -11,52 +11,64 @@ from django.contrib import messages from django.core.exceptions import PermissionDenied from django.shortcuts import redirect -from rest_framework.authentication import TokenAuthentication -from rest_framework.exceptions import AuthenticationFailed -from mfa.helpers import has_mfa -from social_django.middleware import SocialAuthExceptionMiddleware -from social_core import exceptions as social_exceptions from tethys_cli.cli_colors import pretty_output, FG_WHITE from tethys_apps.utilities import get_active_app, user_can_access_app from tethys_portal.views.error import handler_404 - -class TethysSocialAuthExceptionMiddleware(SocialAuthExceptionMiddleware): - def process_exception(self, request, exception): - if hasattr(social_exceptions, exception.__class__.__name__): - if isinstance(exception, social_exceptions.AuthCanceled): - if request.user.is_anonymous: - return redirect("accounts:login") - else: - return redirect("user:settings") - elif isinstance(exception, social_exceptions.AuthAlreadyAssociated): - blurb = "The {0} account you tried to connect to has already been associated with another account." - with pretty_output(FG_WHITE) as p: - p.write(exception.backend.name) - if "google" in exception.backend.name: - blurb = blurb.format("Google") - elif "linkedin" in exception.backend.name: - blurb = blurb.format("LinkedIn") - elif "hydroshare" in exception.backend.name: - blurb = blurb.format("HydroShare") - elif "facebook" in exception.backend.name: - blurb = blurb.format("Facebook") - else: - blurb = blurb.format("social") - - messages.success(request, blurb) - - if request.user.is_anonymous: - return redirect("accounts:login") - else: - return redirect("user:settings") - elif isinstance(exception, social_exceptions.NotAllowedToDisconnect): - blurb = "Unable to disconnect from this social account." - messages.success(request, blurb) - if request.user.is_anonymous: - return redirect("accounts:login") - else: - return redirect("user:settings") +from tethys_portal.optional_dependencies import optional_import, has_module + +# optional imports +has_mfa = optional_import("has_mfa", from_module="mfa.helpers") +SocialAuthExceptionMiddleware = optional_import( + "SocialAuthExceptionMiddleware", from_module="social_django.middleware" +) +social_exceptions = optional_import("exceptions", from_module="social_core") +TokenAuthentication = optional_import( + "TokenAuthentication", from_module="rest_framework.authentication" +) +AuthenticationFailed = optional_import( + "AuthenticationFailed", from_module="rest_framework.exceptions" +) + + +if has_module(SocialAuthExceptionMiddleware): + + class TethysSocialAuthExceptionMiddleware(SocialAuthExceptionMiddleware): + def process_exception(self, request, exception): + if hasattr(social_exceptions, exception.__class__.__name__): + if isinstance(exception, social_exceptions.AuthCanceled): + if request.user.is_anonymous: + return redirect("accounts:login") + else: + return redirect("user:settings") + elif isinstance(exception, social_exceptions.AuthAlreadyAssociated): + blurb = "The {0} account you tried to connect to has already been associated with another account." + with pretty_output(FG_WHITE) as p: + p.write(exception.backend.name) + if "google" in exception.backend.name: + blurb = blurb.format("Google") + elif "linkedin" in exception.backend.name: + blurb = blurb.format("LinkedIn") + elif "hydroshare" in exception.backend.name: + blurb = blurb.format("HydroShare") + elif "facebook" in exception.backend.name: + blurb = blurb.format("Facebook") + else: + blurb = blurb.format("social") + + messages.success(request, blurb) + + if request.user.is_anonymous: + return redirect("accounts:login") + else: + return redirect("user:settings") + elif isinstance(exception, social_exceptions.NotAllowedToDisconnect): + blurb = "Unable to disconnect from this social account." + messages.success(request, blurb) + if request.user.is_anonymous: + return redirect("accounts:login") + else: + return redirect("user:settings") class TethysAppAccessMiddleware: diff --git a/tethys_portal/optional_dependencies.py b/tethys_portal/optional_dependencies.py new file mode 100644 index 000000000..48b7f318a --- /dev/null +++ b/tethys_portal/optional_dependencies.py @@ -0,0 +1,55 @@ +from importlib import import_module + + +class MissingOptionalDependency(ImportError): + pass + + +class FailedImport: + def __init__(self, module, import_error): + self.module_name = module + self.error = import_error + + def __call__(self, *args, **kwargs): + raise MissingOptionalDependency( + f'Optional dependency "{self.module_name}" was not able to be imported because of the following error:\n' + f"{self.error}." + ) + + def __getattr__(self, item): + self.__call__() + + def __getitem__(self, item): + self.__call__() + + +def _attempt_import(module, from_module, error_message): + try: + if from_module: + from_module = import_module(from_module) + return getattr(from_module, module) + return import_module(module) + except ImportError as e: + return FailedImport(module, e) + + +def optional_import(module, from_module=None, error_message=None): + if isinstance(module, (list, tuple)): + return [_attempt_import(m, from_module, error_message) for m in module] + else: + return _attempt_import(module, from_module, error_message) + + +def verify_import(module, error_message=None): + if isinstance(module, FailedImport): + error_message = error_message or ( + f'Optional dependency "{module.module_name}" was not able to be imported because of the ' + f"following error:\n{module.error}." + ) + raise MissingOptionalDependency(error_message) + + +def has_module(module, from_module=None): + if isinstance(module, str): + module = optional_import(module, from_module=from_module) + return not isinstance(module, FailedImport) diff --git a/tethys_portal/settings.py b/tethys_portal/settings.py index d84a303a7..c13ecae55 100644 --- a/tethys_portal/settings.py +++ b/tethys_portal/settings.py @@ -27,13 +27,20 @@ import logging import datetime as dt from pathlib import Path +from importlib import import_module from django.contrib.messages import constants as message_constants + from tethys_apps.utilities import relative_to_tethys_home +from tethys_portal.optional_dependencies import has_module from tethys_cli.cli_colors import write_warning from tethys_cli.gen_commands import generate_secret_key +from tethys_portal.optional_dependencies import optional_import -from bokeh.settings import settings as bokeh_settings, bokehjsdir +# optional imports +bokeh_settings, bokehjsdir = optional_import( + ("settings", "bokehjsdir"), from_module="bokeh.settings" +) log = logging.getLogger(__name__) this_module = sys.modules[__name__] @@ -53,7 +60,8 @@ log.exception( "There was an error while attempting to read the settings from the portal_config.yml file." ) -bokeh_settings.resources = portal_config_settings.pop("BOKEH_RESOURCES", "inline") +if has_module(bokeh_settings): + bokeh_settings.resources = portal_config_settings.pop("BOKEH_RESOURCES", "inline") # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = portal_config_settings.pop("SECRET_KEY", generate_secret_key()) @@ -84,6 +92,8 @@ REGISTER_CONTROLLER = TETHYS_PORTAL_CONFIG.pop("REGISTER_CONTROLLER", None) +ADDITIONAL_URLPATTERNS = TETHYS_PORTAL_CONFIG.pop("ADDITIONAL_URLPATTERNS", []) + SESSION_CONFIG = portal_config_settings.pop("SESSION_CONFIG", {}) # Force user logout once the browser has been closed. # If changed, delete all django_session table entries from the tethys_default database to ensure updated behavior @@ -203,40 +213,49 @@ }, ) +default_installed_apps = [ + "channels", + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django_bootstrap5", + "tethys_apps", + "tethys_compute", + "tethys_config", + "tethys_gizmos", + "tethys_layouts", + "tethys_sdk", + "tethys_services", + "tethys_quotas", + "guardian", +] + +for module in [ + "analytical", + "axes", + "captcha", + "corsheaders", + "django_gravatar", + "django_json_widget", + "mfa", + "oauth2_provider", + "rest_framework", + "rest_framework.authtoken", + "session_security", + "snowpenguin.django.recaptcha2", + "social_django", + "termsandconditions", +]: + if has_module(module): + default_installed_apps.append(module) + + INSTALLED_APPS = portal_config_settings.pop( "INSTALLED_APPS_OVERRIDE", - [ - "channels", - "corsheaders", - "django.contrib.admin", - "django.contrib.auth", - "django.contrib.contenttypes", - "django.contrib.sessions", - "django.contrib.messages", - "django.contrib.staticfiles", - "django_gravatar", - "django_bootstrap5", - "django_json_widget", - "termsandconditions", - "tethys_apps", - "tethys_compute", - "tethys_config", - "tethys_gizmos", - "tethys_layouts", - "tethys_sdk", - "tethys_services", - "tethys_quotas", - "social_django", - "guardian", - "session_security", - "captcha", - "snowpenguin.django.recaptcha2", - "rest_framework", - "rest_framework.authtoken", - "analytical", - "mfa", - "axes", - ], + default_installed_apps, ) INSTALLED_APPS = tuple( @@ -247,28 +266,44 @@ "MIDDLEWARE_OVERRIDE", [ "django.contrib.sessions.middleware.SessionMiddleware", - "corsheaders.middleware.CorsMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "tethys_portal.middleware.TethysMfaRequiredMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", - "tethys_portal.middleware.TethysSocialAuthExceptionMiddleware", "tethys_portal.middleware.TethysAppAccessMiddleware", - "session_security.middleware.SessionSecurityMiddleware", # TODO: Templates need to be upgraded - "axes.middleware.AxesMiddleware", ], ) +if has_module("corsheaders"): + MIDDLEWARE.insert( + MIDDLEWARE.index( + "django.middleware.common.CommonMiddleware" + ), # insert right before + "corsheaders.middleware.CorsMiddleware", + ) +if has_module("social_django"): + MIDDLEWARE.append("tethys_portal.middleware.TethysSocialAuthExceptionMiddleware") +if has_module("session_security"): + MIDDLEWARE.append( + "session_security.middleware.SessionSecurityMiddleware" + ) # TODO: Templates need to be upgraded +if has_module("axes"): + MIDDLEWARE.append("axes.middleware.AxesMiddleware") + MIDDLEWARE = tuple(MIDDLEWARE + portal_config_settings.pop("MIDDLEWARE", [])) +default_authentication_backends = [ + "django.contrib.auth.backends.ModelBackend", + "guardian.backends.ObjectPermissionBackend", +] + +if has_module("axes"): + default_authentication_backends.insert(0, "axes.backends.AxesBackend") + AUTHENTICATION_BACKENDS = portal_config_settings.pop( "AUTHENTICATION_BACKENDS_OVERRIDE", - [ - "axes.backends.AxesBackend", - "django.contrib.auth.backends.ModelBackend", - "guardian.backends.ObjectPermissionBackend", - ], + default_authentication_backends, ) AUTHENTICATION_BACKENDS = tuple( portal_config_settings.pop("AUTHENTICATION_BACKENDS", []) + AUTHENTICATION_BACKENDS @@ -316,29 +351,46 @@ USE_TZ = True +default_context_processors = [ + "django.contrib.auth.context_processors.auth", + "django.template.context_processors.debug", + "django.template.context_processors.i18n", + "django.template.context_processors.media", + "django.template.context_processors.static", + "django.template.context_processors.tz", + "django.template.context_processors.request", + "django.contrib.messages.context_processors.messages", + # "social_django.context_processors.backends", + # "social_django.context_processors.login_redirect", + "tethys_config.context_processors.tethys_global_settings_context", + "tethys_apps.context_processors.tethys_apps_context", + "tethys_gizmos.context_processors.tethys_gizmos_context", + "tethys_portal.context_processors.tethys_portal_context", +] +if has_module("social_django"): + default_context_processors.extend( + [ + "social_django.context_processors.backends", + "social_django.context_processors.login_redirect", + ] + ) + # Templates + +ADDITIONAL_TEMPLATE_DIRS = [ + import_module(d).__path__[0] + for d in TETHYS_PORTAL_CONFIG.get("ADDITIONAL_TEMPLATE_DIRS", []) +] + TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", "DIRS": [ + *ADDITIONAL_TEMPLATE_DIRS, BASE_DIR / "templates", ], "OPTIONS": { - "context_processors": [ - "django.contrib.auth.context_processors.auth", - "django.template.context_processors.debug", - "django.template.context_processors.i18n", - "django.template.context_processors.media", - "django.template.context_processors.static", - "django.template.context_processors.tz", - "django.template.context_processors.request", - "django.contrib.messages.context_processors.messages", - "social_django.context_processors.backends", - "social_django.context_processors.login_redirect", - "tethys_config.context_processors.tethys_global_settings_context", - "tethys_apps.context_processors.tethys_apps_context", - "tethys_gizmos.context_processors.tethys_gizmos_context", - ], + "context_processors": default_context_processors, "loaders": [ "django.template.loaders.filesystem.Loader", "django.template.loaders.app_directories.Loader", @@ -356,8 +408,9 @@ STATICFILES_DIRS = [ BASE_DIR / "static", - bokehjsdir(), ] +if has_module(bokehjsdir): + STATICFILES_DIRS.append(bokehjsdir()) STATICFILES_USE_NPM = TETHYS_PORTAL_CONFIG.pop("STATICFILES_USE_NPM", False) if STATICFILES_USE_NPM: @@ -409,7 +462,7 @@ GRAVATAR_DEFAULT_SIZE = "80" GRAVATAR_DEFAULT_IMAGE = "retro" GRAVATAR_DEFAULT_RATING = "g" -GRAVATAR_DFFAULT_SECURE = True +GRAVATAR_DEFAULT_SECURE = True # OAuth Settings # http://psa.matiasaguirre.net/docs/configuration/index.html @@ -486,9 +539,6 @@ CAPTCHA_CONFIG = portal_config_settings.pop("CAPTCHA_CONFIG", {}) for setting, value in CAPTCHA_CONFIG.items(): setattr(this_module, setting, value) -# If you require reCaptcha to be loaded from somewhere other than https://google.com -# (e.g. to bypass firewall restrictions), you can specify what proxy to use. -# RECAPTCHA_PROXY_HOST: https://recaptcha.net # Placeholders for the ID's required by various web-analytics services supported by Django-Analytical. # Replace False with the tracking ID as a string e.g. SERVICE_ID = 'abcd1234' diff --git a/tethys_portal/templates/analytical_body_bottom.html b/tethys_portal/templates/analytical_body_bottom.html new file mode 100644 index 000000000..f87af1a1d --- /dev/null +++ b/tethys_portal/templates/analytical_body_bottom.html @@ -0,0 +1,2 @@ +{% load analytical %} +{% analytical_body_bottom %} \ No newline at end of file diff --git a/tethys_portal/templates/analytical_body_top.html b/tethys_portal/templates/analytical_body_top.html new file mode 100644 index 000000000..c3f9b3380 --- /dev/null +++ b/tethys_portal/templates/analytical_body_top.html @@ -0,0 +1,2 @@ +{% load analytical %} +{% analytical_body_top %} \ No newline at end of file diff --git a/tethys_portal/templates/analytical_head_bottom.html b/tethys_portal/templates/analytical_head_bottom.html new file mode 100644 index 000000000..702e3a007 --- /dev/null +++ b/tethys_portal/templates/analytical_head_bottom.html @@ -0,0 +1,2 @@ +{% load analytical %} +{% analytical_head_bottom %} \ No newline at end of file diff --git a/tethys_portal/templates/analytical_head_top.html b/tethys_portal/templates/analytical_head_top.html new file mode 100644 index 000000000..07a4a3df9 --- /dev/null +++ b/tethys_portal/templates/analytical_head_top.html @@ -0,0 +1,2 @@ +{% load analytical %} +{% analytical_head_top %} \ No newline at end of file diff --git a/tethys_portal/templates/base.html b/tethys_portal/templates/base.html index 210f74760..620b65648 100644 --- a/tethys_portal/templates/base.html +++ b/tethys_portal/templates/base.html @@ -1,4 +1,4 @@ -{% load static site_settings terms_tags analytical %} +{% load static site_settings %} {# Allows custom attributes to be added to the html tag #} @@ -11,7 +11,9 @@ {# Allows custom attributes to be added to the head tag #} - {% analytical_head_top %} + {% if has_analytical %} + {% include "analytical_head_top.html" %} + {% endif %} {% comment "meta explanation" %} Add custom meta tags to the page. Call block.super to get the default tags @@ -213,21 +215,27 @@ {{ tethys.bootstrap.script_tag|safe }} {% endblock %} + {% if has_session_security %} {% block session_timeout_modal %} {% include 'session_security/all.html' %} {% endblock %} + {% endif %} {% block head %}{% endblock %} {% block extrahead %}{% endblock %} {% block blockbots %}{% endblock %} - {% analytical_head_bottom %} + {% if has_analytical %} + {% include "analytical_head_bottom.html" %} + {% endif %} {# Allows custom attributes to be added to the body tag #} - {% analytical_body_top %} + {% if has_analytical %} + {% include "analytical_body_top.html" %} + {% endif %} {% comment "page explanation" %} The page block allows you to add content to the page. @@ -298,7 +306,9 @@ {% endblock %} {% block tos_override %} - {% show_terms_if_not_agreed %} + {% if has_terms %} + {% include "terms.html" %} + {% endif %} {% endblock %} {% comment "scripts explanation" %} @@ -316,6 +326,8 @@ {% endblock %} - {% analytical_body_bottom %} + {% if has_analytical %} + {% include "analytical_body_bottom.html" %} + {% endif %} diff --git a/tethys_portal/templates/gravatar.html b/tethys_portal/templates/gravatar.html new file mode 100644 index 000000000..48fa9897b --- /dev/null +++ b/tethys_portal/templates/gravatar.html @@ -0,0 +1,7 @@ +{% load gravatar static %} + +{% if gravatar_url %} + +{% else %} +{% if user.email %}{% gravatar user.email image_size %}{% else %}{% gravatar "tethys@example.com" image_size %}{% endif %} +{% endif %} \ No newline at end of file diff --git a/tethys_portal/templates/header.html b/tethys_portal/templates/header.html index 16378e058..449fbb5bf 100644 --- a/tethys_portal/templates/header.html +++ b/tethys_portal/templates/header.html @@ -1,4 +1,4 @@ -{% load gravatar static %} +{% load static %}