fix: docker refactor (apache#31385)

.dockerignore

@@ -45,3 +45,4 @@ superset-frontend/coverage/
 superset/static/assets/
 superset-websocket/dist/
 venv
+.venv

.github/workflows/ephemeral-env.yml

@@ -148,9 +148,20 @@ jobs:
       - name: Setup supersetbot
         uses: ./.github/actions/setup-supersetbot/
 
+      - name: Try to login to DockerHub
+        if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
+        continue-on-error: true
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: Build ephemeral env image
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
         run: |
           supersetbot docker \
             --preset ci \

Dockerfile

@@ -22,33 +22,39 @@ ARG PY_VER=3.10-slim-bookworm
 
 # If BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
 ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
+
+######################################################################
+# superset-node used for building frontend assets
+######################################################################
 FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
+ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
+ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
+ARG DEV_MODE="false"           # Skip frontend build in dev mode
+ENV DEV_MODE=${DEV_MODE}
 
+COPY docker/ /app/docker/
 # Arguments for build configuration
 ARG NPM_BUILD_CMD="build"
-ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
-ARG DEV_MODE="false"           # Skip frontend build in dev mode
-ARG INCLUDE_CHROMIUM="true"    # Include headless Chromium for alerts & reports
-ARG INCLUDE_FIREFOX="false"    # Include headless Firefox if enabled
 
 # Install system dependencies required for node-gyp
-RUN --mount=type=bind,source=./docker,target=/docker \
-    /docker/apt-install.sh build-essential python3 zstd
+RUN /app/docker/apt-install.sh build-essential python3 zstd
 
 # Define environment variables for frontend build
 ENV BUILD_CMD=${NPM_BUILD_CMD} \
     PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
 
 # Run the frontend memory monitoring script
-RUN --mount=type=bind,source=./docker,target=/docker \
-    /docker/frontend-mem-nag.sh
+RUN /app/docker/frontend-mem-nag.sh
 
 WORKDIR /app/superset-frontend
 
 # Create necessary folders to avoid errors in subsequent steps
 RUN mkdir -p /app/superset/static/assets \
              /app/superset/translations
 
+# Copy translation files
+COPY superset/translations /app/superset/translations
+
 # Mount package files and install dependencies if not in dev mode
 RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \
     --mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \
@@ -61,41 +67,28 @@ RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.j
 # Runs the webpack build process
 COPY superset-frontend /app/superset-frontend
 
-
-# Copy translation files
-COPY superset/translations /app/superset/translations
-
 # Build the frontend if not in dev mode
 RUN if [ "$DEV_MODE" = "false" ]; then \
-        BUILD_TRANSLATIONS=$BUILD_TRANSLATIONS npm run ${BUILD_CMD}; \
+        echo "Running 'npm run ${BUILD_CMD}'"; \
+        if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
+            npm run build-translation; \
+        fi; \
+        npm run ${BUILD_CMD}; \
     else \
         echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \
-    fi
-
-# Compile .json files from .po translations (if required) and clean up .po files
-RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
-        npm run build-translation; \
-    else \
-        echo "Skipping translations as requested by build flag"; \
-    fi \
-	# removing translations files regardless
-    && rm -rf /app/superset/translations/*/LC_MESSAGES/*.po \
-              /app/superset/translations/messages.pot
-
+    fi && \
+    rm -rf /app/superset/translations/*/*/*.po
 
-# Transition to Python base image
-FROM python:${PY_VER} AS python-base
-RUN pip install --no-cache-dir --upgrade setuptools pip uv
 
 ######################################################################
-# Final lean image...
+# Base python layer
 ######################################################################
-FROM python-base AS lean
-
-# Build argument for including translations
-ARG BUILD_TRANSLATIONS="false"
+FROM python:${PY_VER} AS python-base
+ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
+ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
+ARG DEV_MODE="false"           # Skip frontend build in dev mode
+ENV DEV_MODE=${DEV_MODE}
 
-WORKDIR /app
 ENV LANG=C.UTF-8 \
     LC_ALL=C.UTF-8 \
     SUPERSET_ENV=production \
@@ -104,126 +97,128 @@ ENV LANG=C.UTF-8 \
     SUPERSET_HOME="/app/superset_home" \
     SUPERSET_PORT=8088
 
+
+RUN useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset
+
+# Some bash scripts needed throughout the layers
+COPY --chmod=755 docker/*.sh /app/docker/
+
+RUN pip install --no-cache-dir --upgrade setuptools pip uv
+
+# Using uv as it's faster/simpler than pip
+RUN uv venv /app/.venv
+ENV PATH="/app/.venv/bin:${PATH}"
+
+# Install Playwright and optionally setup headless browsers
+ARG INCLUDE_CHROMIUM="true"
+ARG INCLUDE_FIREFOX="false"
+RUN --mount=type=cache,target=/root/.cache/pip \
+    if [ "$INCLUDE_CHROMIUM" = "true" ] || [ "$INCLUDE_FIREFOX" = "true" ]; then \
+        pip install playwright && \
+        playwright install-deps && \
+        if [ "$INCLUDE_CHROMIUM" = "true" ]; then playwright install chromium; fi && \
+        if [ "$INCLUDE_FIREFOX" = "true" ]; then playwright install firefox; fi; \
+    else \
+        echo "Skipping browser installation"; \
+    fi
+
+######################################################################
+# Python translation compiler layer
+######################################################################
+FROM python-base AS python-translation-compiler
+
+# Install Python dependencies using docker/pip-install.sh
+COPY requirements/translations.txt requirements/
+RUN --mount=type=cache,target=/root/.cache/pip \
+    /app/docker/pip-install.sh -r requirements/translations.txt
+
+COPY superset/translations/ /app/translations_mo/
+RUN pybabel compile -d /app/translations_mo | true && \
+    rm -f /app/translations_mo/*/*/*.po
+
+######################################################################
+# Python APP common layer
+######################################################################
+FROM python-base AS python-common
+# Copy the entrypoints, make them executable in userspace
+COPY --chmod=755 docker/entrypoints /app/docker/entrypoints
+
+WORKDIR /app
 # Set up necessary directories and user
-RUN --mount=type=bind,source=./docker,target=/docker \
-    mkdir -p ${PYTHONPATH} \
+RUN mkdir -p \
+      ${SUPERSET_HOME} \
+      ${PYTHONPATH} \
       superset/static \
       requirements \
       superset-frontend \
       apache_superset.egg-info \
       requirements \
-    && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
-    && /docker/apt-install.sh \
-        curl \
-        libsasl2-dev \
-        libsasl2-modules-gssapi-mit \
-        libpq-dev \
-        libecpg-dev \
-        libldap2-dev \
-    && touch superset/static/version_info.json \
-    && chown -R superset:superset ./* \
-    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+    && touch superset/static/version_info.json
 
 # Copy required files for Python build
-COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
-COPY --chown=superset:superset superset-frontend/package.json superset-frontend/
-COPY --chown=superset:superset requirements/base.txt requirements/
-COPY --chown=superset:superset scripts/check-env.py scripts/
-
-# Install Python dependencies using docker/pip-install.sh
-RUN --mount=type=bind,source=./docker,target=/docker \
-    --mount=type=cache,target=/root/.cache/pip \
-    /docker/pip-install.sh --requires-build-essential -r requirements/base.txt
-
-# Copy the compiled frontend assets from the node image
-COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
+COPY pyproject.toml setup.py MANIFEST.in README.md ./
+COPY superset-frontend/package.json superset-frontend/
+COPY scripts/check-env.py scripts/
 
-# Copy the main Superset source code
-COPY --chown=superset:superset superset superset
+# keeping for backward compatibility
+COPY --chmod=755 ./docker/entrypoints/run-server.sh /usr/bin/
 
-# Install Superset itself using docker/pip-install.sh
-RUN --mount=type=bind,source=./docker,target=/docker \
-    --mount=type=cache,target=/root/.cache/pip \
-    /docker/pip-install.sh -e .
+# Some debian libs
+RUN /app/docker/apt-install.sh \
+      curl \
+      libsasl2-dev \
+      libsasl2-modules-gssapi-mit \
+      libpq-dev \
+      libecpg-dev \
+      libldap2-dev
 
-# Copy .json translations from the node image
-COPY --chown=superset:superset --from=superset-node /app/superset/translations superset/translations
+# Copy compiled things from previous stages
+COPY --from=superset-node /app/superset/static/assets superset/static/assets
 
-# Compile backend translations and clean up
-COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/
-RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
-        ./scripts/translations/generate_mo_files.sh \
-        && chown -R superset:superset superset/translations; \
-    fi \
-    && rm -rf superset/translations/messages.pot \
-              superset/translations/*/LC_MESSAGES/*.po
+# Merging translations from backend and frontend stages
+COPY --from=superset-node /app/superset/translations superset/translations
+COPY --from=python-translation-compiler /app/translations_mo superset/translations
 
-# Add server run script
-COPY --chmod=755 ./docker/run-server.sh /usr/bin/
-
-# Set user and healthcheck
-USER superset
 HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health"
-
-# Expose port and set CMD
+CMD ["/app/docker/entrypoints/run-server.sh"]
 EXPOSE ${SUPERSET_PORT}
-CMD ["/usr/bin/run-server.sh"]
-
 
 ######################################################################
-# Dev image...
+# Final lean image...
 ######################################################################
-FROM lean AS dev
-
-USER root
+FROM python-common AS lean
+COPY superset superset
 
-# Install dev dependencies
-RUN --mount=type=bind,source=./docker,target=/docker \
-    /docker/apt-install.sh \
-        libnss3 \
-        libdbus-glib-1-2 \
-        libgtk-3-0 \
-        libx11-xcb1 \
-        libasound2 \
-        libxtst6 \
-        git \
-        pkg-config
-
-# Install Playwright and its dependencies
+# Install Python dependencies using docker/pip-install.sh
+COPY requirements/base.txt requirements/
 RUN --mount=type=cache,target=/root/.cache/pip \
-    uv pip install --system playwright \
-    && playwright install-deps
+    /app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt && \
+    uv pip install .
 
-# Optionally install Chromium
-RUN if [ "$INCLUDE_CHROMIUM" = "true" ]; then \
-        playwright install chromium; \
-    else \
-        echo "Skipping Chromium installation in dev mode"; \
-    fi
+RUN python -m compileall /app/superset
 
-# Install GeckoDriver WebDriver and Firefox (if required)
-ARG GECKODRIVER_VERSION=v0.34.0
-ARG FIREFOX_VERSION=125.0.3
-RUN --mount=type=bind,source=./docker,target=/docker \
-    if [ "$INCLUDE_FIREFOX" = "true" ]; then \
-        /docker/apt-install.sh wget bzip2 \
-        && wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
-        && wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
-        && ln -s /opt/firefox/firefox /usr/local/bin/firefox \
-        && apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
-    else \
-        echo "Skipping Firefox installation in dev mode"; \
-    fi
+USER superset
+
+######################################################################
+# Dev image...
+######################################################################
+FROM python-common AS dev
+COPY superset superset
 
-# Install MySQL client dependencies
-RUN --mount=type=bind,source=./docker,target=/docker \
-    /docker/apt-install.sh default-libmysqlclient-dev
+# Debian libs needed for dev
+RUN /app/docker/apt-install.sh \
+    git \
+    pkg-config \
+    default-libmysqlclient-dev
 
 # Copy development requirements and install them
-COPY --chown=superset:superset requirements/development.txt requirements/
-RUN --mount=type=bind,source=./docker,target=/docker \
-    --mount=type=cache,target=/root/.cache/pip \
-    /docker/pip-install.sh --requires-build-essential -r requirements/development.txt
+COPY requirements/*.txt requirements/
+# Install Python dependencies using docker/pip-install.sh
+RUN --mount=type=cache,target=/root/.cache/pip \
+    /app/docker/pip-install.sh --requires-build-essential -r requirements/development.txt && \
+    uv pip install .
+
+RUN python -m compileall /app/superset
 
 USER superset
 
@@ -232,6 +227,4 @@ USER superset
 ######################################################################
 FROM lean AS ci
 
-COPY --chown=superset:superset --chmod=755 ./docker/*.sh /app/docker/
-
-CMD ["/app/docker/docker-ci.sh"]
+CMD ["/app/docker/entrypoints/docker-ci.sh"]

UPDATING.md

@@ -30,6 +30,7 @@ assists people when migrating to a new version.
 - [30099](https://github.com/apache/superset/pull/30099) Translations are no longer included in the default docker image builds. If your environment requires translations, you'll want to set the docker build arg `BUILD_TRANSACTION=true`.
 - [31262](https://github.com/apache/superset/pull/31262) NOTE: deprecated `pylint` in favor of `ruff` as our only python linter. Only affect development workflows positively (not the release itself). It should cover most important rules, be much faster, but some things linting rules that were enforced before may not be enforce in the exact same way as before.
 - [31173](https://github.com/apache/superset/pull/31173) Modified `fetch_csrf_token` to align with HTTP standards, particularly regarding how cookies are handled. If you encounter any issues related to CSRF functionality, please report them as a new issue and reference this PR for context.
+- [31385](https://github.com/apache/superset/pull/31385) Significant docker refactor, reducing access levels for the `superset` user, streamlining layer building, ...
 
 ### Potential Downtime
 

docker/.env

@@ -17,6 +17,7 @@
 
 
 COMPOSE_PROJECT_NAME=superset
+DEV_MODE=true
 
 # database configurations (do not modify)
 DATABASE_DB=superset

docker/docker-bootstrap.sh

@@ -18,6 +18,11 @@
 
 set -eo pipefail
 
+# Make python interactive
+if [ "$DEV_MODE" == "true" ]; then
+    echo "Reinstalling the app in editable mode"
+    uv pip install -e .
+fi
 REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
 # If Cypress run – overwrite the password for admin and export env variables
 if [ "$CYPRESS_CONFIG" == "true" ]; then