diff --git a/.dockerignore b/.dockerignore index e79b8f6e1e78f..33b76412b60cb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -45,3 +45,4 @@ superset-frontend/coverage/ superset/static/assets/ superset-websocket/dist/ venv +.venv diff --git a/.github/workflows/ephemeral-env.yml b/.github/workflows/ephemeral-env.yml index 9b5539ed63736..acf3b0cc72124 100644 --- a/.github/workflows/ephemeral-env.yml +++ b/.github/workflows/ephemeral-env.yml @@ -148,9 +148,20 @@ jobs: - name: Setup supersetbot uses: ./.github/actions/setup-supersetbot/ + - name: Try to login to DockerHub + if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker + continue-on-error: true + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build ephemeral env image env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} run: | supersetbot docker \ --preset ci \ diff --git a/Dockerfile b/Dockerfile index 60ba12eabff32..4ee30930898d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,26 +22,29 @@ ARG PY_VER=3.10-slim-bookworm # If BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise). ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64} + +###################################################################### +# superset-node used for building frontend assets +###################################################################### FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node +ARG BUILD_TRANSLATIONS="false" # Include translations in the final build +ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS} +ARG DEV_MODE="false" # Skip frontend build in dev mode +ENV DEV_MODE=${DEV_MODE} +COPY docker/ /app/docker/ # Arguments for build configuration ARG NPM_BUILD_CMD="build" -ARG BUILD_TRANSLATIONS="false" # Include translations in the final build -ARG DEV_MODE="false" # Skip frontend build in dev mode -ARG INCLUDE_CHROMIUM="true" # Include headless Chromium for alerts & reports -ARG INCLUDE_FIREFOX="false" # Include headless Firefox if enabled # Install system dependencies required for node-gyp -RUN --mount=type=bind,source=./docker,target=/docker \ - /docker/apt-install.sh build-essential python3 zstd +RUN /app/docker/apt-install.sh build-essential python3 zstd # Define environment variables for frontend build ENV BUILD_CMD=${NPM_BUILD_CMD} \ PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true # Run the frontend memory monitoring script -RUN --mount=type=bind,source=./docker,target=/docker \ - /docker/frontend-mem-nag.sh +RUN /app/docker/frontend-mem-nag.sh WORKDIR /app/superset-frontend @@ -49,6 +52,9 @@ WORKDIR /app/superset-frontend RUN mkdir -p /app/superset/static/assets \ /app/superset/translations +# Copy translation files +COPY superset/translations /app/superset/translations + # Mount package files and install dependencies if not in dev mode RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \ --mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \ @@ -61,41 +67,28 @@ RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.j # Runs the webpack build process COPY superset-frontend /app/superset-frontend - -# Copy translation files -COPY superset/translations /app/superset/translations - # Build the frontend if not in dev mode RUN if [ "$DEV_MODE" = "false" ]; then \ - BUILD_TRANSLATIONS=$BUILD_TRANSLATIONS npm run ${BUILD_CMD}; \ + echo "Running 'npm run ${BUILD_CMD}'"; \ + if [ "$BUILD_TRANSLATIONS" = "true" ]; then \ + npm run build-translation; \ + fi; \ + npm run ${BUILD_CMD}; \ else \ echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \ - fi - -# Compile .json files from .po translations (if required) and clean up .po files -RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \ - npm run build-translation; \ - else \ - echo "Skipping translations as requested by build flag"; \ - fi \ - # removing translations files regardless - && rm -rf /app/superset/translations/*/LC_MESSAGES/*.po \ - /app/superset/translations/messages.pot - + fi && \ + rm -rf /app/superset/translations/*/*/*.po -# Transition to Python base image -FROM python:${PY_VER} AS python-base -RUN pip install --no-cache-dir --upgrade setuptools pip uv ###################################################################### -# Final lean image... +# Base python layer ###################################################################### -FROM python-base AS lean - -# Build argument for including translations -ARG BUILD_TRANSLATIONS="false" +FROM python:${PY_VER} AS python-base +ARG BUILD_TRANSLATIONS="false" # Include translations in the final build +ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS} +ARG DEV_MODE="false" # Skip frontend build in dev mode +ENV DEV_MODE=${DEV_MODE} -WORKDIR /app ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ SUPERSET_ENV=production \ @@ -104,126 +97,128 @@ ENV LANG=C.UTF-8 \ SUPERSET_HOME="/app/superset_home" \ SUPERSET_PORT=8088 + +RUN useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset + +# Some bash scripts needed throughout the layers +COPY --chmod=755 docker/*.sh /app/docker/ + +RUN pip install --no-cache-dir --upgrade setuptools pip uv + +# Using uv as it's faster/simpler than pip +RUN uv venv /app/.venv +ENV PATH="/app/.venv/bin:${PATH}" + +# Install Playwright and optionally setup headless browsers +ARG INCLUDE_CHROMIUM="true" +ARG INCLUDE_FIREFOX="false" +RUN --mount=type=cache,target=/root/.cache/pip \ + if [ "$INCLUDE_CHROMIUM" = "true" ] || [ "$INCLUDE_FIREFOX" = "true" ]; then \ + pip install playwright && \ + playwright install-deps && \ + if [ "$INCLUDE_CHROMIUM" = "true" ]; then playwright install chromium; fi && \ + if [ "$INCLUDE_FIREFOX" = "true" ]; then playwright install firefox; fi; \ + else \ + echo "Skipping browser installation"; \ + fi + +###################################################################### +# Python translation compiler layer +###################################################################### +FROM python-base AS python-translation-compiler + +# Install Python dependencies using docker/pip-install.sh +COPY requirements/translations.txt requirements/ +RUN --mount=type=cache,target=/root/.cache/pip \ + /app/docker/pip-install.sh -r requirements/translations.txt + +COPY superset/translations/ /app/translations_mo/ +RUN pybabel compile -d /app/translations_mo | true && \ + rm -f /app/translations_mo/*/*/*.po + +###################################################################### +# Python APP common layer +###################################################################### +FROM python-base AS python-common +# Copy the entrypoints, make them executable in userspace +COPY --chmod=755 docker/entrypoints /app/docker/entrypoints + +WORKDIR /app # Set up necessary directories and user -RUN --mount=type=bind,source=./docker,target=/docker \ - mkdir -p ${PYTHONPATH} \ +RUN mkdir -p \ + ${SUPERSET_HOME} \ + ${PYTHONPATH} \ superset/static \ requirements \ superset-frontend \ apache_superset.egg-info \ requirements \ - && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \ - && /docker/apt-install.sh \ - curl \ - libsasl2-dev \ - libsasl2-modules-gssapi-mit \ - libpq-dev \ - libecpg-dev \ - libldap2-dev \ - && touch superset/static/version_info.json \ - && chown -R superset:superset ./* \ - && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* + && touch superset/static/version_info.json # Copy required files for Python build -COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./ -COPY --chown=superset:superset superset-frontend/package.json superset-frontend/ -COPY --chown=superset:superset requirements/base.txt requirements/ -COPY --chown=superset:superset scripts/check-env.py scripts/ - -# Install Python dependencies using docker/pip-install.sh -RUN --mount=type=bind,source=./docker,target=/docker \ - --mount=type=cache,target=/root/.cache/pip \ - /docker/pip-install.sh --requires-build-essential -r requirements/base.txt - -# Copy the compiled frontend assets from the node image -COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets +COPY pyproject.toml setup.py MANIFEST.in README.md ./ +COPY superset-frontend/package.json superset-frontend/ +COPY scripts/check-env.py scripts/ -# Copy the main Superset source code -COPY --chown=superset:superset superset superset +# keeping for backward compatibility +COPY --chmod=755 ./docker/entrypoints/run-server.sh /usr/bin/ -# Install Superset itself using docker/pip-install.sh -RUN --mount=type=bind,source=./docker,target=/docker \ - --mount=type=cache,target=/root/.cache/pip \ - /docker/pip-install.sh -e . +# Some debian libs +RUN /app/docker/apt-install.sh \ + curl \ + libsasl2-dev \ + libsasl2-modules-gssapi-mit \ + libpq-dev \ + libecpg-dev \ + libldap2-dev -# Copy .json translations from the node image -COPY --chown=superset:superset --from=superset-node /app/superset/translations superset/translations +# Copy compiled things from previous stages +COPY --from=superset-node /app/superset/static/assets superset/static/assets -# Compile backend translations and clean up -COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/ -RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \ - ./scripts/translations/generate_mo_files.sh \ - && chown -R superset:superset superset/translations; \ - fi \ - && rm -rf superset/translations/messages.pot \ - superset/translations/*/LC_MESSAGES/*.po +# Merging translations from backend and frontend stages +COPY --from=superset-node /app/superset/translations superset/translations +COPY --from=python-translation-compiler /app/translations_mo superset/translations -# Add server run script -COPY --chmod=755 ./docker/run-server.sh /usr/bin/ - -# Set user and healthcheck -USER superset HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health" - -# Expose port and set CMD +CMD ["/app/docker/entrypoints/run-server.sh"] EXPOSE ${SUPERSET_PORT} -CMD ["/usr/bin/run-server.sh"] - ###################################################################### -# Dev image... +# Final lean image... ###################################################################### -FROM lean AS dev - -USER root +FROM python-common AS lean +COPY superset superset -# Install dev dependencies -RUN --mount=type=bind,source=./docker,target=/docker \ - /docker/apt-install.sh \ - libnss3 \ - libdbus-glib-1-2 \ - libgtk-3-0 \ - libx11-xcb1 \ - libasound2 \ - libxtst6 \ - git \ - pkg-config - -# Install Playwright and its dependencies +# Install Python dependencies using docker/pip-install.sh +COPY requirements/base.txt requirements/ RUN --mount=type=cache,target=/root/.cache/pip \ - uv pip install --system playwright \ - && playwright install-deps + /app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt && \ + uv pip install . -# Optionally install Chromium -RUN if [ "$INCLUDE_CHROMIUM" = "true" ]; then \ - playwright install chromium; \ - else \ - echo "Skipping Chromium installation in dev mode"; \ - fi +RUN python -m compileall /app/superset -# Install GeckoDriver WebDriver and Firefox (if required) -ARG GECKODRIVER_VERSION=v0.34.0 -ARG FIREFOX_VERSION=125.0.3 -RUN --mount=type=bind,source=./docker,target=/docker \ - if [ "$INCLUDE_FIREFOX" = "true" ]; then \ - /docker/apt-install.sh wget bzip2 \ - && wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \ - && wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \ - && ln -s /opt/firefox/firefox /usr/local/bin/firefox \ - && apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \ - else \ - echo "Skipping Firefox installation in dev mode"; \ - fi +USER superset + +###################################################################### +# Dev image... +###################################################################### +FROM python-common AS dev +COPY superset superset -# Install MySQL client dependencies -RUN --mount=type=bind,source=./docker,target=/docker \ - /docker/apt-install.sh default-libmysqlclient-dev +# Debian libs needed for dev +RUN /app/docker/apt-install.sh \ + git \ + pkg-config \ + default-libmysqlclient-dev # Copy development requirements and install them -COPY --chown=superset:superset requirements/development.txt requirements/ -RUN --mount=type=bind,source=./docker,target=/docker \ - --mount=type=cache,target=/root/.cache/pip \ - /docker/pip-install.sh --requires-build-essential -r requirements/development.txt +COPY requirements/*.txt requirements/ +# Install Python dependencies using docker/pip-install.sh +RUN --mount=type=cache,target=/root/.cache/pip \ + /app/docker/pip-install.sh --requires-build-essential -r requirements/development.txt && \ + uv pip install . + +RUN python -m compileall /app/superset USER superset @@ -232,6 +227,4 @@ USER superset ###################################################################### FROM lean AS ci -COPY --chown=superset:superset --chmod=755 ./docker/*.sh /app/docker/ - -CMD ["/app/docker/docker-ci.sh"] +CMD ["/app/docker/entrypoints/docker-ci.sh"] diff --git a/UPDATING.md b/UPDATING.md index a5ae1736678dd..4267ae340a395 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -30,6 +30,7 @@ assists people when migrating to a new version. - [30099](https://github.com/apache/superset/pull/30099) Translations are no longer included in the default docker image builds. If your environment requires translations, you'll want to set the docker build arg `BUILD_TRANSACTION=true`. - [31262](https://github.com/apache/superset/pull/31262) NOTE: deprecated `pylint` in favor of `ruff` as our only python linter. Only affect development workflows positively (not the release itself). It should cover most important rules, be much faster, but some things linting rules that were enforced before may not be enforce in the exact same way as before. - [31173](https://github.com/apache/superset/pull/31173) Modified `fetch_csrf_token` to align with HTTP standards, particularly regarding how cookies are handled. If you encounter any issues related to CSRF functionality, please report them as a new issue and reference this PR for context. +- [31385](https://github.com/apache/superset/pull/31385) Significant docker refactor, reducing access levels for the `superset` user, streamlining layer building, ... ### Potential Downtime diff --git a/docker/.env b/docker/.env index 57575da76edc1..7511766569340 100644 --- a/docker/.env +++ b/docker/.env @@ -17,6 +17,7 @@ COMPOSE_PROJECT_NAME=superset +DEV_MODE=true # database configurations (do not modify) DATABASE_DB=superset diff --git a/docker/docker-bootstrap.sh b/docker/docker-bootstrap.sh index 2f0b29ce34716..1a4e04be94e7a 100755 --- a/docker/docker-bootstrap.sh +++ b/docker/docker-bootstrap.sh @@ -18,6 +18,11 @@ set -eo pipefail +# Make python interactive +if [ "$DEV_MODE" == "true" ]; then + echo "Reinstalling the app in editable mode" + uv pip install -e . +fi REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt" # If Cypress run – overwrite the password for admin and export env variables if [ "$CYPRESS_CONFIG" == "true" ]; then diff --git a/docker/docker-ci.sh b/docker/entrypoints/docker-ci.sh similarity index 100% rename from docker/docker-ci.sh rename to docker/entrypoints/docker-ci.sh diff --git a/docker/run-server.sh b/docker/entrypoints/run-server.sh similarity index 100% rename from docker/run-server.sh rename to docker/entrypoints/run-server.sh diff --git a/docker/pip-install.sh b/docker/pip-install.sh index 7e69a6efba164..7deb4fa19a7dd 100755 --- a/docker/pip-install.sh +++ b/docker/pip-install.sh @@ -47,10 +47,10 @@ fi # Choose whether to use pip cache if $USE_CACHE; then echo "Using pip cache..." - uv pip install --system "${ARGS[@]}" + uv pip install "${ARGS[@]}" else echo "Disabling pip cache..." - uv pip install --system --no-cache-dir "${ARGS[@]}" + uv pip install --no-cache-dir "${ARGS[@]}" fi # Remove build-essential if it was installed diff --git a/requirements/translations.in b/requirements/translations.in new file mode 100644 index 0000000000000..98f65931c4c84 --- /dev/null +++ b/requirements/translations.in @@ -0,0 +1 @@ +babel diff --git a/requirements/translations.txt b/requirements/translations.txt new file mode 100644 index 0000000000000..4eab2d21f4f08 --- /dev/null +++ b/requirements/translations.txt @@ -0,0 +1,9 @@ +# SHA1:cad160f3d4cd7c33896f42a479eeaa1b5bedc5fb +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +babel==2.16.0 + # via -r requirements/translations.in