Switch database server to PostgreSQL 13

.github/workflows/ci.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: docker-compose up -d
-      - name: Wait for MySQL
+      - name: Wait for PostgreSQL
         run: |
           set +e
           for i in {0..60}; do
-            docker-compose exec -T chris_store_dev_db mysqladmin -uroot -prootp status 2> /dev/null
+            docker-compose exec -T chris_store_dev_db psql -U chris -d chris_store_dev -c "select 1" 2> /dev/null
             if [ "$?" = "0" ]; then
               dbup=y
               break
@@ -29,11 +29,9 @@ jobs:
             echo .
           done
           if [ "$dbup" != "y" ]; then
-            echo "::error ::Timed out waiting for MySQL database."
+            echo "::error ::Timed out waiting for PostgreSQL database."
             exit 1
           fi
-      - name: Prepare MySQL for tests
-        run: docker-compose exec -T chris_store_dev_db mysql -uroot -prootp -e 'GRANT ALL PRIVILEGES ON *.* TO "chris"@"%"'
       - name: Run all tests
         run: docker-compose exec -T chris_store_dev python manage.py test
       - run: docker-compose down -v

Dockerfile

@@ -49,10 +49,10 @@ RUN apt-get update                                               \
   && export LC_ALL=en_US.UTF-8                                   \
   && locale-gen en_US.UTF-8                                      \
   && dpkg-reconfigure locales                                    \
-  && apt-get install -y libssl-dev libmysqlclient-dev            \
+  && apt-get install -y libssl-dev libpq-dev                     \
   && apt-get install -y apache2 apache2-dev                      \
-  && pip install --upgrade pip                                    \
-  && pip install -r ${REQPATH}/${ENVIRONMENT}.txt                    \
+  && pip install --upgrade pip                                   \
+  && pip install -r ${REQPATH}/${ENVIRONMENT}.txt                \
   && useradd -l -u $UID -ms /bin/bash localuser
 
 # Start as user localuser

README.md

@@ -3,7 +3,7 @@
 ![License][license-badge]
 ![Last Commit][last-commit-badge]
 
-Back end for the ChRIS store. This is a Django-MySQL project that houses descriptions of ChRIS plugin-apps and workflows for registering to a ChRIS CUBE instance.
+Back end for the ChRIS store. This is a Django-PostgreSQL project that houses descriptions of ChRIS plugin-apps and workflows for registering to a ChRIS CUBE instance.
 
 ## ChRIS store development, testing and deployment
 

docker-compose_dev.yml

@@ -35,17 +35,15 @@ services:
       role: "Development server"
 
   chris_store_dev_db:
-    image: mysql:8
-    command: --collation-server=utf8mb4_0900_as_cs
+    image: postgres:13
     volumes:
-      - chris_store_dev_db_data:/var/lib/mysql:z
+      - chris_store_dev_db_data:/var/lib/postgresql/data:z
     environment:
-      - MYSQL_ROOT_PASSWORD=rootp
-      - MYSQL_DATABASE=chris_store_dev
-      - MYSQL_USER=chris
-      - MYSQL_PASSWORD=Chris1234
+      - POSTGRES_DB=chris_store_dev
+      - POSTGRES_USER=chris
+      - POSTGRES_PASSWORD=Chris1234
     labels:
-      name: "ChRIS_store MySQL Database"
+      name: "ChRIS_store PostgreSQL Database"
       role: "Development database"
 
   swift_service_dev:

docker-entrypoint.sh

@@ -2,9 +2,9 @@
 
 if [[ "$DJANGO_DB_MIGRATE" == 'on' ]]; then
   if [[ "$DJANGO_SETTINGS_MODULE" == 'config.settings.local' ]]; then
-    python migratedb.py -u root -p rootp --host chris_store_dev_db --noinput
+    python migratedb.py -u chris -p Chris1234 -d chris_store_dev --host chris_store_dev_db --noinput
   elif [[ "$DJANGO_SETTINGS_MODULE" == 'config.settings.production' ]]; then
-    python migratedb.py -u root -p $MYSQL_ROOT_PASSWORD --host $DATABASE_HOST --noinput
+    python migratedb.py -u $POSTGRES_USER -p $POSTGRES_PASSWORD -d $POSTGRES_DB --host $DATABASE_HOST --noinput
   fi
 fi
 

make.sh

@@ -42,8 +42,8 @@ if [[ "$1" == 'up' ]]; then
 
     title -d 1 "Pulling core containers where needed..."
         printf "${LightCyan}%40s${Green}%-40s${Yellow}\n"                   \
-                "docker pull" " mysql:8"                                    | ./boxes.sh
-        docker pull mysql:8                                                 | ./boxes.sh
+                "docker pull" " postgres:13"                                    | ./boxes.sh
+        docker pull postgres:13                                                 | ./boxes.sh
         echo ""                                                             | ./boxes.sh
         printf "${LightCyan}%40s${Green}%-40s${Yellow}\n"                   \
                 "docker pull " "fnndsc/docker-swift-onlyone"                | ./boxes.sh
@@ -64,14 +64,14 @@ if [[ "$1" == 'up' ]]; then
     windowBottom
 
     title -d 1 "Waiting until ChRIS store database server is ready to accept connections..."
-        docker-compose -f docker-compose_dev.yml exec chris_store_dev_db sh -c 'while ! mysqladmin -uroot -prootp status 2> /dev/null; do sleep 5; done;' >& dc.out > /dev/null
+        echo "This might take a few seconds..."                          | ./boxes.sh ${Yellow}
+        windowBottom
+        docker-compose -f docker-compose_dev.yml exec chris_store_dev_db sh -c 'while ! psql -U chris -d chris_store_dev -c "select 1" 2> /dev/null; do sleep 5; done;' >& dc.out > /dev/null
         echo -en "\033[2A\033[2K"
-        sed -E 's/[[:alnum:]]+:/\n&/g' dc.out | ./boxes.sh
-        # Give all permissions to chris user on the test DB. This is required for the Django tests:
-        echo "Granting <chris> user all DB permissions...."             | ./boxes.sh ${LightCyan}
-        echo "This is required for the Django tests."                   | ./boxes.sh ${LightCyan}
-        docker-compose -f docker-compose_dev.yml exec chris_store_dev_db mysql -uroot -prootp -e 'GRANT ALL PRIVILEGES ON test_chris_store_dev.* TO "chris"@"%"'  >& dc.out > /dev/null
-        cat dc.out                                                      | ./boxes.sh
+        cat dc.out                                                      | ./boxes.sh ${LightGreen}
+        echo ""                                                         | ./boxes.sh
+        echo "ChRIS store database is ready to accept connections"      | ./boxes.sh ${LightGreen}
+        echo ""                                                         | ./boxes.sh
     windowBottom
 
     title -d 1 "Running Django Unit tests..."

requirements/base.txt

@@ -2,8 +2,7 @@ Django==2.2.24
 django-filter==2.4.0
 djangorestframework==3.12.4
 django-cors-middleware==1.5.0
-mysqlclient==2.0.3
-mysql-connector-python==8.0.25
+psycopg2==2.8.6
 python-swiftclient==3.12.0
 django-storage-swift==1.2.19
 mod-wsgi==4.8.0

store_backend/config/settings/common.py

@@ -98,7 +98,7 @@
 
 DATABASES = {
     'default': {
-        'ENGINE': 'mysql.connector.django',
+        'ENGINE': 'django.db.backends.postgresql',
     }
 }
 

store_backend/config/settings/local.py

@@ -81,7 +81,7 @@
 DATABASES['default']['PASSWORD'] = 'Chris1234'
 DATABASES['default']['TEST'] = {'NAME': 'test_chris_store_dev'}
 DATABASES['default']['HOST'] = 'chris_store_dev_db'
-DATABASES['default']['PORT'] = '3306'
+DATABASES['default']['PORT'] = '5432'
 
 # Mail settings
 # ------------------------------------------------------------------------------

store_backend/config/settings/production.py

@@ -43,9 +43,9 @@ def get_secret(setting, secret_type=env):
 # DATABASE CONFIGURATION
 # ------------------------------------------------------------------------------
 # Raises ImproperlyConfigured exception if DATABASE_URL not in os.environ
-DATABASES['default']['NAME'] = get_secret('MYSQL_DATABASE')
-DATABASES['default']['USER'] = get_secret('MYSQL_USER')
-DATABASES['default']['PASSWORD'] = get_secret('MYSQL_PASSWORD')
+DATABASES['default']['NAME'] = get_secret('POSTGRES_DB')
+DATABASES['default']['USER'] = get_secret('POSTGRES_USER')
+DATABASES['default']['PASSWORD'] = get_secret('POSTGRES_PASSWORD')
 DATABASES['default']['HOST'] = get_secret('DATABASE_HOST')
 DATABASES['default']['PORT'] = get_secret('DATABASE_PORT')
 

store_backend/migratedb.py

@@ -2,7 +2,7 @@
 
 import time
 import sys
-import MySQLdb
+import psycopg2
 from argparse import ArgumentParser
 
 # django needs to be loaded
@@ -15,6 +15,7 @@
 parser = ArgumentParser(description="Check database service connection")
 parser.add_argument('-u', '--user', help="Database user name")
 parser.add_argument('-p', '--password', help="Database user password")
+parser.add_argument('-d', '--database', help="Database name")
 parser.add_argument('--host', help="Database host")
 parser.add_argument('--max-attempts', type=int, dest='attempts',
                     help="Maximum number of connection attempts")
@@ -30,9 +31,8 @@
 db = None
 while max_tries > 0 and db is None:
     try:
-        db = MySQLdb.connect(user=args.user,
-                     passwd=args.password,
-                     host=host)
+        db = psycopg2.connect(host=host, user=args.user, password=args.password,
+                              dbname=args.database)
     except Exception:
         time.sleep(5)
         max_tries -= 1

swarm/prod/docker-compose.yml

@@ -15,7 +15,7 @@ services:
     image: fnndsc/chris_store
     environment:
       - DATABASE_HOST=chris_store_db
-      - DATABASE_PORT=3306
+      - DATABASE_PORT=5432
       - SWIFT_AUTH_URL=http://swift_service:8080/auth/v1.0
     env_file:
       - ./secrets/.chris_store.env
@@ -34,18 +34,17 @@ services:
       role: "Production server using Apache's mod_wsgi"
 
   chris_store_db:
-    image: mysql:8
-    command: --collation-server=utf8mb4_0900_as_cs
+    image: postgres:13
     volumes:
-      - chris_store_db_data:/var/lib/mysql
+      - chris_store_db_data:/var/lib/postgresql/data
     env_file:
       - ./secrets/.chris_store_db.env
     deploy:
       restart_policy:
         condition: on-failure
     labels:
-      name: "ChRIS_store MySQL Database"
-      role: "Production MySQL database"
+      name: "ChRIS_store PostgreSQL Database"
+      role: "Production PostgreSQL database"
 
   swift_service:
     image: fnndsc/docker-swift-onlyone