Merge remote-tracking branch 'origin/master' into feature/404-view

backend/alembic/versions/0009_models_refactor.py

@@ -80,13 +80,13 @@ def upgrade() -> None:
             "url_screenshots",
             existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"),
             nullable=True,
-            existing_server_default=sa.text("'[]'"),
+            existing_server_default=sa.text("(JSON_ARRAY())"),
         )
         batch_op.alter_column(
             "path_screenshots",
             existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"),
             nullable=True,
-            existing_server_default=sa.text("'[]'"),
+            existing_server_default=sa.text("(JSON_ARRAY())"),
         )
 
     try:
@@ -127,13 +127,13 @@ def downgrade() -> None:
             "path_screenshots",
             existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"),
             nullable=False,
-            existing_server_default=sa.text("'[]'"),
+            existing_server_default=sa.text("(JSON_ARRAY())"),
         )
         batch_op.alter_column(
             "url_screenshots",
             existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"),
             nullable=False,
-            existing_server_default=sa.text("'[]'"),
+            existing_server_default=sa.text("(JSON_ARRAY())"),
         )
         batch_op.alter_column(
             "file_size_units", existing_type=mysql.VARCHAR(length=10), nullable=True

backend/alembic/versions/0014_asset_files.py

@@ -33,8 +33,8 @@
 
 
 def migrate_to_mysql() -> None:
-    if ROMM_DB_DRIVER != "mariadb":
-        raise Exception("Version 3.0 requires MariaDB as database driver!")
+    if ROMM_DB_DRIVER not in ("mariadb", "mysql"):
+        raise Exception("Version 3.0 requires MariaDB or MySQL as database driver!")
 
     # Skip if sqlite database is not mounted
     if not os.path.exists(f"{SQLITE_DB_BASE_PATH}/romm.db"):

backend/alembic/versions/1.8.1_.py

@@ -20,11 +20,19 @@ def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     with op.batch_alter_table("roms") as batch_op:
         batch_op.add_column(
-            sa.Column("url_screenshots", sa.JSON(), nullable=False, server_default="[]")
+            sa.Column(
+                "url_screenshots",
+                sa.JSON(),
+                nullable=False,
+                server_default=sa.text("(JSON_ARRAY())"),
+            )
         )
         batch_op.add_column(
             sa.Column(
-                "path_screenshots", sa.JSON(), nullable=False, server_default="[]"
+                "path_screenshots",
+                sa.JSON(),
+                nullable=False,
+                server_default=sa.text("(JSON_ARRAY())"),
             )
         )
     # ### end Alembic commands ###

backend/config/__init__.py

@@ -25,12 +25,13 @@ def str_to_bool(value: str) -> bool:
 ASSETS_BASE_PATH: Final = f"{ROMM_BASE_PATH}/assets"
 FRONTEND_RESOURCES_PATH: Final = "/assets/romm/resources"
 
-# MARIADB
+# DATABASE
 DB_HOST: Final = os.environ.get("DB_HOST", "127.0.0.1")
 DB_PORT: Final = int(os.environ.get("DB_PORT", 3306))
 DB_USER: Final = os.environ.get("DB_USER")
 DB_PASSWD: Final = os.environ.get("DB_PASSWD")
 DB_NAME: Final = os.environ.get("DB_NAME", "romm")
+ROMM_DB_DRIVER: Final = os.environ.get("ROMM_DB_DRIVER", "mariadb")
 
 # REDIS
 REDIS_HOST: Final = os.environ.get("REDIS_HOST", "127.0.0.1")
@@ -62,9 +63,6 @@ def str_to_bool(value: str) -> bool:
 # MOBYGAMES
 MOBYGAMES_API_KEY: Final = os.environ.get("MOBYGAMES_API_KEY", "")
 
-# DB DRIVERS
-ROMM_DB_DRIVER: Final = os.environ.get("ROMM_DB_DRIVER", "mariadb")
-
 # AUTH
 ROMM_AUTH_SECRET_KEY: Final = os.environ.get(
     "ROMM_AUTH_SECRET_KEY", secrets.token_hex(32)
@@ -79,6 +77,7 @@ def str_to_bool(value: str) -> bool:
 # OIDC
 OIDC_ENABLED: Final = str_to_bool(os.environ.get("OIDC_ENABLED", "false"))
 OIDC_PROVIDER: Final = os.environ.get("OIDC_PROVIDER", "")
+OIDC_CREATE_USER: Final = str_to_bool(os.environ.get("OIDC_CREATE_USER", "false"))
 OIDC_CLIENT_ID: Final = os.environ.get("OIDC_CLIENT_ID", "")
 OIDC_CLIENT_SECRET: Final = os.environ.get("OIDC_CLIENT_SECRET", "")
 OIDC_REDIRECT_URI: Final = os.environ.get("OIDC_REDIRECT_URI", "")

backend/config/config_manager.py

@@ -83,30 +83,34 @@ def get_db_engine() -> URL:
             str: database connection string
         """
 
-        if ROMM_DB_DRIVER == "mariadb":
-            if not DB_USER or not DB_PASSWD:
-                log.critical(
-                    "Missing database credentials, check your environment variables!"
-                )
-                sys.exit(3)
-
-            return URL.create(
-                drivername="mariadb+mariadbconnector",
-                username=DB_USER,
-                password=DB_PASSWD,
-                host=DB_HOST,
-                port=DB_PORT,
-                database=DB_NAME,
-            )
-
         # DEPRECATED
         if ROMM_DB_DRIVER == "sqlite":
             log.critical("Sqlite is not supported anymore, migrate to mariaDB")
             sys.exit(6)
         # DEPRECATED
 
-        log.critical(f"{ROMM_DB_DRIVER} database not supported")
-        sys.exit(3)
+        if ROMM_DB_DRIVER == "mariadb":
+            driver = "mariadb+mariadbconnector"
+        elif ROMM_DB_DRIVER == "mysql":
+            driver = "mysql+mysqlconnector"
+        else:
+            log.critical(f"{ROMM_DB_DRIVER} database not supported")
+            sys.exit(3)
+
+        if not DB_USER or not DB_PASSWD:
+            log.critical(
+                "Missing database credentials, check your environment variables!"
+            )
+            sys.exit(3)
+
+        return URL.create(
+            drivername=driver,
+            username=DB_USER,
+            password=DB_PASSWD,
+            host=DB_HOST,
+            port=DB_PORT,
+            database=DB_NAME,
+        )
 
     def _parse_config(self):
         """Parses each entry in the config.yml"""

backend/handler/auth/base_handler.py

@@ -4,7 +4,12 @@
 from typing import Any, Final, Optional
 
 import httpx
-from config import OIDC_ENABLED, OIDC_SERVER_APPLICATION_URL, ROMM_AUTH_SECRET_KEY
+from config import (
+    OIDC_CREATE_USER,
+    OIDC_ENABLED,
+    OIDC_SERVER_APPLICATION_URL,
+    ROMM_AUTH_SECRET_KEY,
+)
 from exceptions.auth_exceptions import OAuthCredentialsException, UserDisabledException
 from fastapi import HTTPException, status
 from joserfc import jwt
@@ -247,6 +252,7 @@ async def validate_token(self, token: str) -> jwt.Token:
 
     async def get_current_active_user_from_openid_token(self, token: Any):
         from handler.database import db_user_handler
+        from models.user import Role, User
 
         if not OIDC_ENABLED:
             return None, None
@@ -279,11 +285,21 @@ async def get_current_active_user_from_openid_token(self, token: Any):
 
         user = db_user_handler.get_user_by_email(email)
         if user is None:
-            log.error("User with email '%s' not found", email)
-            raise HTTPException(
-                status_code=status.HTTP_404_NOT_FOUND,
-                detail="User not found",
-            )
+            if OIDC_CREATE_USER:
+                log.info("User with email '%s' not found, creating new user", email)
+                user = User(
+                    username=email.split("@")[0],
+                    email=email,
+                    enabled=True,
+                    role=Role.VIEWER,
+                )
+                db_user_handler.add_user(user)
+            else:
+                log.error("User with email '%s' not found", email)
+                raise HTTPException(
+                    status_code=status.HTTP_404_NOT_FOUND,
+                    detail="User not found",
+                )
 
         if not user.enabled:
             raise UserDisabledException

docker/Dockerfile

@@ -1,5 +1,5 @@
 # Stages:
-#   - front-build-stage: Build frontend
+#   - frontend-build: Build frontend
 #   - backend-build: Build backend environment
 #   - rahasher-build: Build RAHasher
 #   - nginx-build: Build nginx modules
@@ -13,7 +13,7 @@ ARG NODE_VERSION=20.16
 ARG PYTHON_VERSION=3.12
 
 
-FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} AS front-build-stage
+FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} AS frontend-build
 WORKDIR /front
 
 COPY ./frontend/package*.json ./
@@ -99,10 +99,20 @@ RUN git clone https://github.com/evanmiller/mod_zip.git && \
 FROM nginx:${NGINX_VERSION}-alpine${ALPINE_VERSION} AS production-stage
 ARG WEBSERVER_FOLDER=/var/www/html
 
+# Install required packages and dependencies
+RUN apk add --no-cache \
+    bash \
+    libmagic \
+    mariadb-connector-c \
+    p7zip \
+    python3 \
+    tzdata \
+    valkey
+
 COPY --from=rahasher-build /RALibretro/bin64/RAHasher /usr/bin/RAHasher
 COPY --from=nginx-build ./nginx/objs/ngx_http_zip_module.so /usr/lib/nginx/modules/
 
-COPY --from=front-build-stage /front/dist ${WEBSERVER_FOLDER}
+COPY --from=frontend-build /front/dist ${WEBSERVER_FOLDER}
 COPY ./frontend/assets/default ${WEBSERVER_FOLDER}/assets/default
 COPY ./frontend/assets/emulatorjs ${WEBSERVER_FOLDER}/assets/emulatorjs
 COPY ./frontend/assets/ruffle ${WEBSERVER_FOLDER}/assets/ruffle
@@ -113,17 +123,6 @@ RUN mkdir -p ${WEBSERVER_FOLDER}/assets/romm && \
     ln -s /romm/resources ${WEBSERVER_FOLDER}/assets/romm/resources && \
     ln -s /romm/assets ${WEBSERVER_FOLDER}/assets/romm/assets
 
-# Install required packages and dependencies
-RUN apk add --no-cache \
-    bash \
-    libmagic \
-    mariadb-connector-c \
-    p7zip \
-    python3 \
-    tini \
-    tzdata \
-    valkey
-
 COPY ./backend /backend
 
 # Setup init script and config files
@@ -152,5 +151,5 @@ VOLUME ["/romm/resources", "/romm/library", "/romm/assets", "/romm/config", "/re
 EXPOSE 8080 6379/tcp
 WORKDIR /romm
 
-ENTRYPOINT ["/sbin/tini", "--", "/docker-entrypoint.sh"]
+ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["/init"]

docker/init_scripts/init

@@ -115,19 +115,43 @@ watchdog_process_pid() {
 	fi
 }
 
+stop_process_pid() {
+	PROCESS=$1
+	if [[ -f "/tmp/${PROCESS}.pid" ]]; then
+		PID=$(cat "/tmp/${PROCESS}.pid") || true
+		if [[ -d "/proc/${PID}" ]]; then
+			info_log "stopping ${PROCESS}"
+			kill "${PID}" || true
+			# wait for process exit
+			while [ -e "/proc/${PID}" ]; do sleep 0.1; done
+		fi
+	fi
+}
+
+shutdown() {
+	# shutdown in reverse order
+	stop_process_pid scheduler
+	stop_process_pid worker
+	stop_process_pid watcher
+	stop_process_pid nginx
+	stop_process_pid gunicorn
+	stop_process_pid valkey-server
+}
+
 # switch to backend directory
 cd /backend || { error_log "/backend directory doesn't seem to exist"; }
 
 info_log "Starting up, please wait..."
 
+# setup trap handler
+exited=0
+trap 'exited=1 && shutdown' SIGINT SIGTERM EXIT
+
 # clear any leftover PID files
 rm /tmp/*.pid -f
 
 # function definition done, lets start our main loop
-while true; do
-	# check for died processes every 5 seconds
-	sleep 5
-
+while ! ((exited)); do
 	# Start Valkey server if we dont have a corresponding PID file
 	# and REDIS_HOST is not set (which would mean we're using an external Redis/Valkey)
 	if [[ -z ${REDIS_HOST:=""} ]]; then
@@ -138,7 +162,7 @@ while true; do
 	# but only if it was not successful since the last full docker container start
 	if [[ ${ALEMBIC_SUCCESS:="false"} == "false" ]]; then
 		if alembic upgrade head; then
-			debug_log "database schema migrations suceeded"
+			debug_log "database schema migrations succeeded"
 			ALEMBIC_SUCCESS="true"
 		else
 			error_log "Something went horribly wrong with our database"
@@ -165,4 +189,7 @@ while true; do
 	watchdog_process_pid python worker
 	# Start scheduler if we dont have a corresponding PID file
 	watchdog_process_pid python scheduler
+
+	# check for died processes every 5 seconds
+	sleep 5
 done

env.template

@@ -37,6 +37,7 @@ AUTHENTIK_BOOTSTRAP_PASSWORD=
 ROMM_AUTH_SECRET_KEY=
 OIDC_ENABLED=
 OIDC_PROVIDER=
+OIDC_CREATE_USER=false
 OIDC_CLIENT_ID=
 OIDC_CLIENT_SECRET=
 OIDC_REDIRECT_URI=