diff --git a/.github/jobs/bash_functions.sh b/.github/jobs/bash_functions.sh new file mode 100755 index 00000000..05cffc94 --- /dev/null +++ b/.github/jobs/bash_functions.sh @@ -0,0 +1,34 @@ +#! /bin/bash + +# utility function to run command get log the time it took to run +# if CMD_LOGFILE is set, send output to that file and unset var +function time_command { + local start_seconds=$SECONDS + echo "RUNNING: $*" + + local error + # pipe output to log file if set + if [ "x$CMD_LOGFILE" == "x" ]; then + "$@" + error=$? + else + echo "Logging to ${CMD_LOGFILE}" + "$@" &>> $CMD_LOGFILE + error=$? + unset CMD_LOGFILE + fi + + local duration=$(( SECONDS - start_seconds )) + echo "TIMING: Command took `printf '%02d' $(($duration / 60))`:`printf '%02d' $(($duration % 60))` (MM:SS): '$*'" + if [ ${error} -ne 0 ]; then + echo "ERROR: '$*' exited with status = ${error}" + fi + return $error +} + +# utility function to construct the DockerHub tag name to be used, +# replacing slashes with underscores in the branch name + +function get_dockerhub_tag { + echo ${DOCKERHUB_REPO}:$(echo ${SOURCE_BRANCH} | sed 's%/%_%g') +} diff --git a/.github/jobs/build_docker_image.sh b/.github/jobs/build_docker_image.sh new file mode 100755 index 00000000..d8e0e963 --- /dev/null +++ b/.github/jobs/build_docker_image.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh + +DOCKERHUB_TAG=$(get_dockerhub_tag) + +DOCKERFILE_PATH=${GITHUB_WORKSPACE}/internal/scripts/docker/Dockerfile.copy + +CMD_LOGFILE=${GITHUB_WORKSPACE}/docker_build.log + +time_command docker build -t ${DOCKERHUB_TAG} \ + --build-arg SOURCE_BRANCH \ + --build-arg MET_BASE_REPO \ + --build-arg MET_BASE_TAG \ + -f $DOCKERFILE_PATH ${GITHUB_WORKSPACE} +if [ $? != 0 ]; then + cat ${CMD_LOGFILE} + exit 1 +fi + +# Copy the dist directory from the image +id=$(docker create ${DOCKERHUB_TAG}) +time_command docker cp $id:/METviewer/dist dist +docker rm -v $id diff --git a/.github/jobs/configure_sonarqube.sh b/.github/jobs/configure_sonarqube.sh new file mode 100755 index 00000000..8e3d4fa0 --- /dev/null +++ b/.github/jobs/configure_sonarqube.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Constants +SONAR_PROPERTIES_DIR=internal/scripts/sonarqube +SONAR_PROPERTIES=sonar-project.properties + +# Check that this is being run from the top-level METviewer directory +if [ ! -e $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES ]; then + echo "ERROR: ${0} -> must be run from the top-level METviewer directory" + exit 1 +fi + +# Check required environment variables +if [ -z ${SOURCE_BRANCH+x} ]; then + echo "ERROR: ${0} -> \$SOURCE_BRANCH not defined!" + exit 1 +fi +if [ -z ${WD_REFERENCE_BRANCH+x} ]; then + echo "ERROR: ${0} -> \$WD_REFERENCE_BRANCH not defined!" + exit 1 +fi +if [ -z ${SONAR_HOST_URL+x} ]; then + echo "ERROR: ${0} -> \$SONAR_HOST_URL not defined!" + exit 1 +fi +if [ -z ${SONAR_TOKEN+x} ]; then + echo "ERROR: ${0} -> \$SONAR_TOKEN not defined!" + exit 1 +fi + +# Define the version string +export SONAR_PROJECT_VERSION=$(cat docs/version) + +# +# Define the $SONAR_REFERENCE_BRANCH as the +# - Target of any requests +# - Manual setting for workflow dispatch +# - Source branch for any pushes (e.g. develop) +# +if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then + export SONAR_REFERENCE_BRANCH=$GITHUB_BASE_REF +elif [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then + export SONAR_REFERENCE_BRANCH=$WD_REFERENCE_BRANCH +else + export SONAR_REFERENCE_BRANCH=$SOURCE_BRANCH +fi + +# Configure the sonar-project.properties +[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES +sed -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_BRANCH_NAME|$SOURCE_BRANCH|" \ + $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES + +# Define new code when the source and reference branches differ +if [ "$SOURCE_BRANCH" != "$SONAR_REFERENCE_BRANCH" ]; then + echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES +fi + +echo "Contents of the $SONAR_PROPERTIES file:" +cat $SONAR_PROPERTIES diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 86b368fd..dec23969 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,6 +11,9 @@ - [ ] Will this PR result in changes to the test suite? **[Yes or No]**
If **yes**, describe the new output and/or changes to the existing output:
+- [ ] Do these changes introduce new SonarQube findings? **[Yes or No]**
+If **yes**, please describe: + - [ ] Please complete this pull request review by **[Fill in date]**.
## Pull Request Checklist ## diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml new file mode 100644 index 00000000..3e85000e --- /dev/null +++ b/.github/workflows/sonarqube.yml @@ -0,0 +1,99 @@ +name: SonarQube Scan + +# Run SonarQube for Pull Requests and changes to the develop and main_vX.Y branches + +on: + + # Trigger analysis for pushes to develop and main_vX.Y branches + push: + branches: + - develop + - 'main_v**' + paths-ignore: + - 'docs/**' + - '.github/pull_request_template.md' + - '.github/ISSUE_TEMPLATE/**' + - '**/README.md' + - '**/LICENSE.md' + + # Trigger analysis for pull requests to develop and main_vX.Y branches + pull_request: + types: [opened, synchronize, reopened] + branches: + - develop + - 'main_v**' + paths-ignore: + - 'docs/**' + - '.github/pull_request_template.md' + - '.github/ISSUE_TEMPLATE/**' + - '**/README.md' + - '**/LICENSE.md' + + workflow_dispatch: + inputs: + reference_branch: + description: 'Reference Branch' + default: develop + type: string + +env: + DOCKERHUB_REPO: dtcenter/metviewer-dev + +jobs: + sonarqube: + name: SonarQube Scan + runs-on: ubuntu-latest + + steps: + + - uses: actions/checkout@v4 + with: + # Disable shallow clones for better analysis + fetch-depth: 0 + + - name: Create output directories + run: mkdir -p ${RUNNER_WORKSPACE}/logs + + - name: Get branch name + id: get_branch_name + run: echo branch_name=${GITHUB_REF#refs/heads/} >> $GITHUB_OUTPUT + + - name: Build METviewer in Docker + run: .github/jobs/build_docker_image.sh + env: + SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }} + + - name: Configure SonarQube + run: .github/jobs/configure_sonarqube.sh + env: + SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }} + WD_REFERENCE_BRANCH: ${{ github.event.inputs.reference_branch }} + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: SonarQube Scan + uses: sonarsource/sonarqube-scan-action@master + env: + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: SonarQube Quality Gate check + id: sonarqube-quality-gate-check + uses: sonarsource/sonarqube-quality-gate-action@master + # Force to fail step after specific time. + timeout-minutes: 5 + env: + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: Copy log files into logs directory + if: always() + run: cp ${GITHUB_WORKSPACE}/*.log ${RUNNER_WORKSPACE}/logs/ + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: logs_sonarqube + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore diff --git a/docker/hooks/build b/docker/hooks/build deleted file mode 100644 index 631634c8..00000000 --- a/docker/hooks/build +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -docker build -t $IMAGE_NAME \ ---build-arg METVIEWER_GIT_NAME=v6.0.0-beta1 \ ---build-arg METCALCPY_GIT_NAME=v3.0.0-beta1 \ ---build-arg METDATAIO_GIT_NAME=v3.0.0-beta1 \ ---build-arg METPLOTPY_GIT_NAME=v3.0.0-beta1 . - diff --git a/docker/Dockerfile b/internal/scripts/docker/Dockerfile similarity index 80% rename from docker/Dockerfile rename to internal/scripts/docker/Dockerfile index a3f02588..0b57e51b 100644 --- a/docker/Dockerfile +++ b/internal/scripts/docker/Dockerfile @@ -3,24 +3,14 @@ FROM centos:7 MAINTAINER Tatiana Burek # -# This Dockerfile checks out METviewer from GitHub and builds the specified branch or tag. +# This Dockerfile checks out METviewer and its dependencies from GitHub and builds the specified branch or tag. +# Use the develop branches for dependencies by default but override with "--build-arg". # -ENV METVIEWER_GIT_NAME v6.0.0-beta2 -ENV METCALCPY_GIT_NAME v3.0.0-beta2 -ENV METPLOTPY_GIT_NAME v3.0.0-beta2 -ENV METDATAIO_GIT_NAME v3.0.0-beta2 -# -# Constants -# -ENV TOMCAT_MINOR_VERSION 5.61 -ENV TOMCAT_MAJOR_VERSION 8 -ENV TOMCAT_VERSION ${TOMCAT_MAJOR_VERSION}.${TOMCAT_MINOR_VERSION} - -ENV METVIEWER_GIT_URL https://github.com/dtcenter/METviewer -ENV METCALCPY_GIT_URL https://github.com/dtcenter/METcalcpy -ENV METPLOTPY_GIT_URL https://github.com/dtcenter/METplotpy -ENV METDATAIO_GIT_URL https://github.com/dtcenter/METdataio +ARG METVIEWER_GIT_NAME +ARG METPLOTPY_GIT_NAME=develop +ARG METCALCPY_GIT_NAME=develop +ARG METDATAIO_GIT_NAME=develop # # METVIEWER_GIT_NAME is required. @@ -28,39 +18,27 @@ ENV METDATAIO_GIT_URL https://github.com/dtcenter/METdataio RUN if [ "x${METVIEWER_GIT_NAME}" = "x" ]; then \ echo "ERROR: METVIEWER_GIT_NAME undefined! Rebuild with \"--build-arg METVIEWER_GIT_NAME={branch, tag, or hash}\""; \ exit 1; \ - else \ - echo "Build Argument METVIEWER_GIT_NAME=${METVIEWER_GIT_NAME}"; \ - fi + fi -# -# METCALCPY_GIT_NAME is required. -# -RUN if [ "x${METCALCPY_GIT_NAME}" = "x" ]; then \ - echo "ERROR: METCALCPY_GIT_NAME undefined! Rebuild with \"--build-arg METCALCPY_GIT_NAME={branch, tag, or hash}\""; \ - exit 1; \ - else \ - echo "Build Argument METCALCPY_GIT_NAME=${METCALCPY_GIT_NAME}"; \ - fi +RUN echo "Build Argument METVIEWER_GIT_NAME=${METVIEWER_GIT_NAME}" \ + && echo "Build Argument METPLOTPY_GIT_NAME=${METPLOTPY_GIT_NAME}" \ + && echo "Build Argument METCALCPY_GIT_NAME=${METCALCPY_GIT_NAME}" \ + && echo "Build Argument METDATAIO_GIT_NAME=${METDATAIO_GIT_NAME}" # -# METPLOTPY_GIT_NAME is required. +# Repository URLs # -RUN if [ "x${METPLOTPY_GIT_NAME}" = "x" ]; then \ - echo "ERROR: METPLOTPY_GIT_NAME undefined! Rebuild with \"--build-arg METPLOTPY_GIT_NAME={branch, tag, or hash}\""; \ - exit 1; \ - else \ - echo "Build Argument METPLOTPY_GIT_NAME=${METPLOTPY_GIT_NAME}"; \ - fi +ENV METVIEWER_GIT_URL https://github.com/dtcenter/METviewer +ENV METPLOTPY_GIT_URL https://github.com/dtcenter/METplotpy +ENV METCALCPY_GIT_URL https://github.com/dtcenter/METcalcpy +ENV METDATAIO_GIT_URL https://github.com/dtcenter/METdataio # -# METDATAIO_GIT_NAME is required. +# Constants # -RUN if [ "x${METDATAIO_GIT_NAME}" = "x" ]; then \ - echo "ERROR: METDATAIO_GIT_NAME undefined! Rebuild with \"--build-arg METDATAIO_GIT_NAME={branch, tag, or hash}\""; \ - exit 1; \ - else \ - echo "Build Argument METDATAIO_GIT_NAME=${METDATAIO_GIT_NAME}"; \ - fi +ENV TOMCAT_MINOR_VERSION 5.61 +ENV TOMCAT_MAJOR_VERSION 8 +ENV TOMCAT_VERSION ${TOMCAT_MAJOR_VERSION}.${TOMCAT_MINOR_VERSION} # # Install system updates @@ -123,17 +101,14 @@ RUN mkdir /METviewer-python \ WORKDIR /METviewer-python/ RUN git clone --branch ${METCALCPY_GIT_NAME} ${METCALCPY_GIT_URL} - RUN echo "Checking out METplotpy ${METPLOTPY_GIT_NAME} from ${METPLOTPY_GIT_URL}" WORKDIR /METviewer-python/ RUN git clone --branch ${METPLOTPY_GIT_NAME} ${METPLOTPY_GIT_URL} - -RUN echo "Checking out METdataio ${METDATAIO_GIT_NAME} from ${METDATAIO_GIT_URL}" \ +RUN echo "Checking out METdataio ${METDATAIO_GIT_NAME} from ${METDATAIO_GIT_URL}" WORKDIR /METviewer-python/ RUN git clone --branch ${METDATAIO_GIT_NAME} ${METDATAIO_GIT_URL} - # # Install METviewer # @@ -221,7 +196,7 @@ RUN ln -sf /usr/local/bin/pip3.10 /usr/bin/pip3 RUN ln -sf /usr/bin/pip3 /usr/bin/pip # -# install GEOS - needed for cartopy +# Install GEOS - needed for cartopy # WORKDIR /tmp RUN wget http://download.osgeo.org/geos/geos-3.7.2.tar.bz2 @@ -268,31 +243,30 @@ RUN pip install cartopy \ && pip install python-dateutil==2.8.2 \ && pip install opencv-python \ && pip install pandas==1.5.2 -# -# - - # -# set env vars +# Set env vars # ENV PYTHONPATH "${PYTHONPATH}:/METviewer-python/METcalcpy/:/METviewer-python/METplotpy/" ENV METPLOTPY_BASE "/METviewer-python/METplotpy/" -# remove unneeded scripts +# +# Remove unneeded scripts +# RUN rm /METviewer/bin/auto_test.sh \ - && rm /METviewer/bin/mv_test.sh \ - && rm /METviewer/bin/nightly_test.sh \ - && rm /METviewer/bin/prep_dist.sh \ - && rm /METviewer/bin/mv_compare.sh \ - && rm -r /METviewer/test + && rm /METviewer/bin/mv_test.sh \ + && rm /METviewer/bin/nightly_test.sh \ + && rm /METviewer/bin/prep_dist.sh \ + && rm /METviewer/bin/mv_compare.sh \ + && rm -r /METviewer/test -# change permissions of the scripts +# +# Change permissions of the scripts +# RUN chmod 755 /METviewer/bin/mv_batch.sh \ - && chmod 755 /METviewer/bin/mv_load.sh \ - && chmod 755 /METviewer/bin/mv_prune.sh \ - && chmod 755 /METviewer/bin/mv_scorecard.sh - + && chmod 755 /METviewer/bin/mv_load.sh \ + && chmod 755 /METviewer/bin/mv_prune.sh \ + && chmod 755 /METviewer/bin/mv_scorecard.sh ENTRYPOINT ${CATALINA_HOME}/bin/startup.sh && /bin/bash CMD ["true"] diff --git a/docker/Dockerfile_for_Singularity b/internal/scripts/docker/Dockerfile.apptainer similarity index 80% rename from docker/Dockerfile_for_Singularity rename to internal/scripts/docker/Dockerfile.apptainer index 007ce009..4f18ba12 100644 --- a/docker/Dockerfile_for_Singularity +++ b/internal/scripts/docker/Dockerfile.apptainer @@ -3,27 +3,14 @@ FROM centos:7 MAINTAINER Tatiana Burek # -# This Dockerfile checks out METviewer from GitHub and builds the specified branch or tag. +# This Dockerfile checks out METviewer and its dependencies from GitHub and builds the specified branch or tag. +# Use the develop branches for dependencies by default but override with "--build-arg". # -ARG METVIEWER_GIT_NAME -ARG METCALCPY_GIT_NAME -ARG METPLOTPY_GIT_NAME -ARG METDATAIO_GIT_NAME - -# -# Constants -# -ENV TOMCAT_MINOR_VERSION 5.61 -ENV TOMCAT_MAJOR_VERSION 8 -ENV TOMCAT_VERSION ${TOMCAT_MAJOR_VERSION}.${TOMCAT_MINOR_VERSION} - -ENV METVIEWER_GIT_URL https://github.com/dtcenter/METviewer -ENV METCALCPY_GIT_URL https://github.com/dtcenter/METcalcpy -ENV METPLOTPY_GIT_URL https://github.com/dtcenter/METplotpy -ENV METDATAIO_GIT_URL https://github.com/dtcenter/METdataio -# this umask is needed for Tomcat -ENV UMASK 002 +ARG METVIEWER_GIT_NAME +ARG METPLOTPY_GIT_NAME=develop +ARG METCALCPY_GIT_NAME=develop +ARG METDATAIO_GIT_NAME=develop # # METVIEWER_GIT_NAME is required. @@ -31,39 +18,27 @@ ENV UMASK 002 RUN if [ "x${METVIEWER_GIT_NAME}" = "x" ]; then \ echo "ERROR: METVIEWER_GIT_NAME undefined! Rebuild with \"--build-arg METVIEWER_GIT_NAME={branch, tag, or hash}\""; \ exit 1; \ - else \ - echo "Build Argument METVIEWER_GIT_NAME=${METVIEWER_GIT_NAME}"; \ fi -# -# METCALCPY_GIT_NAME is required. -# -RUN if [ "x${METCALCPY_GIT_NAME}" = "x" ]; then \ - echo "ERROR: METCALCPY_GIT_NAME undefined! Rebuild with \"--build-arg METCALCPY_GIT_NAME={branch, tag, or hash}\""; \ - exit 1; \ - else \ - echo "Build Argument METCALCPY_GIT_NAME=${METCALCPY_GIT_NAME}"; \ - fi +RUN echo "Build Argument METVIEWER_GIT_NAME=${METVIEWER_GIT_NAME}" \ + && echo "Build Argument METPLOTPY_GIT_NAME=${METPLOTPY_GIT_NAME}" \ + && echo "Build Argument METCALCPY_GIT_NAME=${METCALCPY_GIT_NAME}" \ + && echo "Build Argument METDATAIO_GIT_NAME=${METDATAIO_GIT_NAME}" # -# METPLOTPY_GIT_NAME is required. +# Repository URLs # -RUN if [ "x${METPLOTPY_GIT_NAME}" = "x" ]; then \ - echo "ERROR: METPLOTPY_GIT_NAME undefined! Rebuild with \"--build-arg METPLOTPY_GIT_NAME={branch, tag, or hash}\""; \ - exit 1; \ - else \ - echo "Build Argument METPLOTPY_GIT_NAME=${METPLOTPY_GIT_NAME}"; \ - fi +ENV METVIEWER_GIT_URL https://github.com/dtcenter/METviewer +ENV METPLOTPY_GIT_URL https://github.com/dtcenter/METplotpy +ENV METCALCPY_GIT_URL https://github.com/dtcenter/METcalcpy +ENV METDATAIO_GIT_URL https://github.com/dtcenter/METdataio # -# METDATAIO_GIT_NAME is required. +# Constants # -RUN if [ "x${METDATAIO_GIT_NAME}" = "x" ]; then \ - echo "ERROR: METDATAIO_GIT_NAME undefined! Rebuild with \"--build-arg METDATAIO_GIT_NAME={branch, tag, or hash}\""; \ - exit 1; \ - else \ - echo "Build Argument METDATAIO_GIT_NAME=${METDATAIO_GIT_NAME}"; \ - fi +ENV TOMCAT_MINOR_VERSION 5.61 +ENV TOMCAT_MAJOR_VERSION 8 +ENV TOMCAT_VERSION ${TOMCAT_MAJOR_VERSION}.${TOMCAT_MINOR_VERSION} # # Install system updates @@ -89,8 +64,7 @@ RUN yum -y install wget tar git ant R mysql ksh \ && rm -rf /var/cache/yum/* \ && yum clean all -RUN yum install java-1.8.0-openjdk-devel - +RUN yum install java-1.8.0-openjdk-devel # # Install gsl-2.5 on which the R gsl package depends. @@ -138,13 +112,11 @@ RUN mkdir /METviewer-python \ WORKDIR /METviewer-python/ RUN git clone --branch ${METCALCPY_GIT_NAME} ${METCALCPY_GIT_URL} - RUN echo "Checking out METplotpy ${METPLOTPY_GIT_NAME} from ${METPLOTPY_GIT_URL}" WORKDIR /METviewer-python/ RUN git clone --branch ${METPLOTPY_GIT_NAME} ${METPLOTPY_GIT_URL} - -RUN echo "Checking out METdataio ${METDATAIO_GIT_NAME} from ${METDATAIO_GIT_URL}" \ +RUN echo "Checking out METdataio ${METDATAIO_GIT_NAME} from ${METDATAIO_GIT_URL}" WORKDIR /METviewer-python/ RUN git clone --branch ${METDATAIO_GIT_NAME} ${METDATAIO_GIT_URL} @@ -202,7 +174,6 @@ RUN git clone --branch ${METVIEWER_GIT_NAME} ${METVIEWER_GIT_URL} /METviewer \ > mv_prune.sh-DOCKER \ && mv mv_prune.sh-DOCKER mv_prune.sh - # # Install Python 3.10.4 # @@ -235,9 +206,8 @@ RUN ln -sf /usr/bin/python3 /usr/bin/python RUN ln -sf /usr/local/bin/pip3.10 /usr/bin/pip3 RUN ln -sf /usr/bin/pip3 /usr/bin/pip - # -# install GEOS - needed for cartopy +# Install GEOS - needed for cartopy # WORKDIR /tmp RUN wget http://download.osgeo.org/geos/geos-3.7.2.tar.bz2 @@ -249,7 +219,6 @@ RUN ldconfig WORKDIR /tmp RUN rm -r geos-3.7.2.tar.bz2 - # # Install Python packages # @@ -286,34 +255,32 @@ RUN pip install cartopy \ && pip install opencv-python \ && pip install pandas==1.5.2 - - # -# set env vars +# Set env vars # ENV PYTHONPATH "${PYTHONPATH}:/METviewer-python/METcalcpy/:/METviewer-python/METplotpy/" ENV METPLOTPY_BASE "/METviewer-python/METplotpy/" - -# remove unneeded scripts +# +# Remove unneeded scripts +# RUN rm /METviewer/bin/auto_test.sh \ - && rm /METviewer/bin/mv_test.sh \ - && rm /METviewer/bin/nightly_test.sh \ - && rm /METviewer/bin/prep_dist.sh \ - && rm /METviewer/bin/mv_compare.sh \ - && rm -r /METviewer/test + && rm /METviewer/bin/mv_test.sh \ + && rm /METviewer/bin/nightly_test.sh \ + && rm /METviewer/bin/prep_dist.sh \ + && rm /METviewer/bin/mv_compare.sh \ + && rm -r /METviewer/test # -# Change permission on exe's +# Change permissions of the scripts # RUN chmod 755 /METviewer/bin/mv_batch.sh \ - && chmod 755 /METviewer/bin/mv_load.sh \ - && chmod 755 /METviewer/bin/mv_prune.sh \ - && chmod 755 /METviewer/bin/mv_scorecard.sh - + && chmod 755 /METviewer/bin/mv_load.sh \ + && chmod 755 /METviewer/bin/mv_prune.sh \ + && chmod 755 /METviewer/bin/mv_scorecard.sh # -# database install +# Database install # # @@ -321,9 +288,9 @@ RUN chmod 755 /METviewer/bin/mv_batch.sh \ # COPY fix-permissions.sh ./ RUN chmod 777 ./fix-permissions.sh -RUN ./fix-permissions.sh /var/lib/mysql/ && \ - ./fix-permissions.sh /var/log/mariadb/ && \ - ./fix-permissions.sh /var/run/ +RUN ./fix-permissions.sh /var/lib/mysql/ \ + && ./fix-permissions.sh /var/log/mariadb/ \ + && ./fix-permissions.sh /var/run/ COPY docker-entrypoint.sh / RUN chmod 777 /docker-entrypoint.sh diff --git a/internal/scripts/docker/Dockerfile.copy b/internal/scripts/docker/Dockerfile.copy new file mode 100644 index 00000000..8b73beea --- /dev/null +++ b/internal/scripts/docker/Dockerfile.copy @@ -0,0 +1,284 @@ +FROM centos:7 + +MAINTAINER Tatiana Burek + +# +# This Dockerfile checks out the METviewer dependencies from GitHub and builds the local METviewer repository. +# Use the develop branches for dependencies by default but override with "--build-arg". +# + +ARG SOURCE_BRANCH +ARG METPLOTPY_GIT_NAME=develop +ARG METCALCPY_GIT_NAME=develop +ARG METDATAIO_GIT_NAME=develop + +# +# SOURCE_BRANCH is required to define the branch of the local METviewer repository. +# +RUN if [ "x${SOURCE_BRANCH}" = "x" ]; then \ + echo "ERROR: SOURCE_BRANCH undefined! Rebuild with \"--build-arg SOURCE_BRANCH={branch name}\""; \ + exit 1; \ + else \ + echo "Build Argument SOURCE_BRANCH=${SOURCE_BRANCH}"; \ + fi +ENV METVIEWER_GIT_NAME ${SOURCE_BRANCH} + +RUN echo "Build Argument METVIEWER_GIT_NAME=${METVIEWER_GIT_NAME}" \ + && echo "Build Argument METPLOTPY_GIT_NAME=${METPLOTPY_GIT_NAME}" \ + && echo "Build Argument METCALCPY_GIT_NAME=${METCALCPY_GIT_NAME}" \ + && echo "Build Argument METDATAIO_GIT_NAME=${METDATAIO_GIT_NAME}" + +# +# Repository URLs +# +ENV METVIEWER_GIT_URL https://github.com/dtcenter/METviewer +ENV METPLOTPY_GIT_URL https://github.com/dtcenter/METplotpy +ENV METCALCPY_GIT_URL https://github.com/dtcenter/METcalcpy +ENV METDATAIO_GIT_URL https://github.com/dtcenter/METdataio + +# +# Constants +# +ENV TOMCAT_MINOR_VERSION 5.61 +ENV TOMCAT_MAJOR_VERSION 8 +ENV TOMCAT_VERSION ${TOMCAT_MAJOR_VERSION}.${TOMCAT_MINOR_VERSION} + +# +# Install system updates +# +RUN yum -y update \ + && yum -y install epel-release \ + && yum -y install 'dnf-command(config-manager)' \ + && yum-config-manager --enable PowerTools + +# +# Install required packages +# +RUN yum -y install wget tar git ant java R mysql ksh \ + && rm -rf /var/cache/yum/* \ + && yum clean all + +# +# Install gsl-2.5 on which the R gsl package depends. +# The centos7 gal package is too old (version 1.5). +# +RUN echo "Compiling gsl-2.5" \ + && curl -SL http://gnu.askapache.com/gsl/gsl-2.5.tar.gz | tar zxC /lib \ + && cd /lib/gsl-2.5 \ + && ./configure --prefix=/usr --libdir=/usr/lib64 >& configure.log \ + && make >& make.log \ + && make install >& make_install.log + +# +# Setup default cran repo +# +RUN echo "r <- getOption('repos'); r['CRAN'] <- 'http://cran.us.r-project.org'; options(repos = r);" > ~/.Rprofile + +# +# Install required R packages +# +RUN Rscript -e "install.packages('boot')" \ + && Rscript -e "install.packages('plotrix')" \ + && Rscript -e "install.packages('gsl')" \ + && Rscript -e "install.packages('data.table')" \ + && Rscript -e "install.packages('verification')" + +# +# Install Tomcat +# +ENV CATALINA_HOME /opt/tomcat + +RUN wget https://archive.apache.org/dist/tomcat/tomcat-${TOMCAT_MAJOR_VERSION}/v${TOMCAT_VERSION}/bin/apache-tomcat-${TOMCAT_VERSION}.tar.gz \ + && tar -xvf apache-tomcat-${TOMCAT_VERSION}.tar.gz \ + && rm apache-tomcat*.tar.gz \ + && mv apache-tomcat* ${CATALINA_HOME} \ + && chmod +x ${CATALINA_HOME}/bin/*sh + +EXPOSE 8080 + +# +# Install METplus python components +# +RUN mkdir /METviewer-python \ + && echo "Checking out METcalcpy ${METCALCPY_GIT_NAME} from ${METCALCPY_GIT_URL}" +WORKDIR /METviewer-python/ +RUN git clone --branch ${METCALCPY_GIT_NAME} ${METCALCPY_GIT_URL} + +RUN echo "Checking out METplotpy ${METPLOTPY_GIT_NAME} from ${METPLOTPY_GIT_URL}" +WORKDIR /METviewer-python/ +RUN git clone --branch ${METPLOTPY_GIT_NAME} ${METPLOTPY_GIT_URL} + +RUN echo "Checking out METdataio ${METDATAIO_GIT_NAME} from ${METDATAIO_GIT_URL}" +WORKDIR /METviewer-python/ +RUN git clone --branch ${METDATAIO_GIT_NAME} ${METDATAIO_GIT_URL} + +# +# Copy in the local METviewer repository +# +RUN echo "Copying METviewer into /METviewer" \ + && mkdir -p /METviewer + +COPY . /METviewer + +RUN if [ ! -e "/METviewer/build.xml" ]; then \ + echo "ERROR: docker build must be run from the top-level METviewer directory"; \ + exit 1; \ + fi + +RUN echo "Configuring and building METviewer" \ + && cd /METviewer \ + && cat webapp/metviewer/WEB-INF/classes/build.properties | \ + sed -r 's%db.host=.*%db.host=mysql_mv%g' | \ + sed -r 's%db.user=.*%db.user=root%g' | \ + sed -r 's%db.password=.*%db.password=mvuser%g' | \ + sed -r 's%db.management.system=.*%db.management.system=mysql%g' | \ + sed -r 's%output.dir=.*%output.dir=/opt/tomcat/webapps/metviewer_output/%g' | \ + sed -r 's%webapps.dir=.*%webapps.dir=/opt/tomcat/webapps/metviewer/%g' | \ + sed -r 's%url.output=.*%url.output=http://localhost:8080/metviewer_output/%g' | \ + sed -r 's%python.env=.*%python.env=/usr/%g' | \ + sed -r 's%metcalcpy.home=.*%metcalcpy.home=/METviewer-python/METcalcpy/%g' | \ + sed -r 's%metplotpy.home=.*%metplotpy.home=/METviewer-python/METplotpy/%g' \ + > build.properties \ + && ant -Dbuild.properties.file=./build.properties \ + -Ddb.management.system=mysql \ + -Dmetcalcpy.path=/METviewer-python/METcalcpy/ \ + -Dmetplotpy.path=/METviewer-python/METplotpy/ \ + -Dpython.env.path=/usr/ war \ + && mv /METviewer/dist/*.war ${CATALINA_HOME}/webapps \ + && echo "Configuring METviewer scripts" \ + && cd /METviewer/bin \ + && cat mv_batch.sh | \ + sed -r 's%JAVA=.*%JAVA=java\nMV_HOME=/METviewer%g' | \ + sed -r 's%PYTHON_ENV=.*%PYTHON_ENV=/usr%g' | \ + sed -r 's%METCALCPY_HOME=.*%METCALCPY_HOME=/METviewer-python/METcalcpy/%g' | \ + sed -r 's%METPLOTPY_HOME=.*%METPLOTPY_HOME=/METviewer-python/METplotpy/%g' \ + > mv_batch.sh-DOCKER \ + && mv mv_batch.sh-DOCKER mv_batch.sh \ + && cat mv_load.sh | \ + sed -r 's%PYTHON_ENV=.*%PYTHON_ENV=/usr%g' | \ + sed -r 's%METDATAIO_HOME=.*%METDATAIO_HOME=/METviewer-python/METdataio/%g' \ + > mv_load.sh-DOCKER \ + && mv mv_load.sh-DOCKER mv_load.sh \ + && cat mv_scorecard.sh | \ + sed -r 's%JAVA=.*%JAVA=java\nMV_HOME=/METviewer%g' | \ + sed -r 's%PYTHON_ENV=.*%PYTHON_ENV=/usr%g' | \ + sed -r 's%METCALCPY_HOME=.*%METCALCPY_HOME=/METviewer-python/METcalcpy/%g' | \ + sed -r 's%METPLOTPY_HOME=.*%METPLOTPY_HOME=/METviewer-python/METplotpy/%g' \ + > mv_scorecard.sh-DOCKER \ + && mv mv_scorecard.sh-DOCKER mv_scorecard.sh \ + && cat mv_prune.sh | \ + sed -r 's%JAVA=.*%JAVA=java\nMV_HOME=/METviewer%g' | \ + sed -r 's%PYTHON_ENV=.*%PYTHON_ENV=/usr%g' | \ + sed -r 's%METCALCPY_HOME=.*%METCALCPY_HOME=/METviewer-python/METcalcpy/%g' | \ + sed -r 's%METPLOTPY_HOME=.*%METPLOTPY_HOME=/METviewer-python/METplotpy/%g' \ + > mv_prune.sh-DOCKER \ + && mv mv_prune.sh-DOCKER mv_prune.sh + +# +# Install Python 3.10.4 +# +RUN yum install gcc openssl11 openssl11-devel libreadline-gplv2-dev libncursesw5-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev openssl-devel libssl-dev bzip2-devel libffi-devel zlib-devel libproj-dev proj-data proj-bin libgeos-dev bzip2 -y +RUN mkdir /usr/local/openssl11 +WORKDIR /usr/local/openssl11 +RUN ln -s /usr/lib64/openssl11 lib +RUN ln -s /usr/include/openssl11 include + +RUN curl https://www.python.org/ftp/python/3.10.4/Python-3.10.4.tgz --output /tmp/Python-3.10.4.tgz +WORKDIR /tmp +RUN tar xzf Python-3.10.4.tgz +WORKDIR /tmp/Python-3.10.4 +RUN ./configure --enable-optimizations --with-openssl=/usr/local/openssl11 +RUN yum install make -y +RUN make altinstall +RUN yum install which -y +WORKDIR /tmp +RUN rm -r Python-3.10.4.tgz +RUN yum -y install epel-release +RUN curl -sS https://bootstrap.pypa.io/get-pip.py -o get-pip.py | python3.10 +RUN python3.10 -m pip install --trusted-host pypi.python.org --trusted-host files.pythonhosted.org --trusted-host pypi.org --upgrade pip + +# +# Create a link for python3 +# +RUN ln -sf /usr/local/bin/python3.10 /usr/bin/python3 +RUN ln -sf /usr/bin/python3 /usr/bin/python + +RUN ln -sf /usr/local/bin/pip3.10 /usr/bin/pip3 +RUN ln -sf /usr/bin/pip3 /usr/bin/pip + +# +# Install GEOS - needed for cartopy +# +WORKDIR /tmp +RUN wget http://download.osgeo.org/geos/geos-3.7.2.tar.bz2 +RUN tar xjf geos-3.7.2.tar.bz2 +WORKDIR /tmp/geos-3.7.2 +RUN ./configure --enable-php; make clean ; make +RUN make install +RUN ldconfig +WORKDIR /tmp +RUN rm -r geos-3.7.2.tar.bz2 + +# +# Install Python packages +# +RUN pip install cartopy \ + && pip install eofs \ + && pip install imutils==0.5.4 \ + && pip install imageio==2.19.2 \ + && pip install lxml==4.9.1 \ + && pip install matplotlib==3.5.2 \ + && pip install netcdf4==1.6.2 \ + && pip install numpy==1.22.0 \ + && pip install pytest==7.1.2 \ + && pip install metpy==1.3.1 \ + && pip install pyyaml==6.0 \ + && pip install scikit-image==0.19.3 \ + && pip install scikit-learn \ + && pip install scipy==1.11.1 \ + && pip install xarray==2022.3.0 \ + && pip install PyMySQL==1.0.2 \ + && pip install pint==0.19.2 \ + && pip install plotly==5.9.0 \ + && pip install kaleido==0.2.1 \ + && pip install attrs==22.1.0 \ + && pip install exceptiongroup==1.0.4 \ + && pip install iniconfig==1.1.1 \ + && pip install packaging==22.0 \ + && pip install pluggy==1.0.0 \ + && pip install pytz==2022.6 \ + && pip install setuptools==65.5.1 \ + && pip install six==1.16.0 \ + && pip install tomli==2.0.1 \ + && pip install wheel==0.38.1 \ + && pip install python-dateutil==2.8.2 \ + && pip install opencv-python \ + && pip install pandas==1.5.2 + +# +# Set env vars +# +ENV PYTHONPATH "${PYTHONPATH}:/METviewer-python/METcalcpy/:/METviewer-python/METplotpy/" +ENV METPLOTPY_BASE "/METviewer-python/METplotpy/" + +# +# Remove unneeded scripts +# +RUN rm /METviewer/bin/auto_test.sh \ + && rm /METviewer/bin/mv_test.sh \ + && rm /METviewer/bin/nightly_test.sh \ + && rm /METviewer/bin/prep_dist.sh \ + && rm /METviewer/bin/mv_compare.sh \ + && rm -r /METviewer/test + +# +# Change permissions of the scripts +# +RUN chmod 755 /METviewer/bin/mv_batch.sh \ + && chmod 755 /METviewer/bin/mv_load.sh \ + && chmod 755 /METviewer/bin/mv_prune.sh \ + && chmod 755 /METviewer/bin/mv_scorecard.sh + +ENTRYPOINT ${CATALINA_HOME}/bin/startup.sh && /bin/bash +CMD ["true"] + diff --git a/docker/README b/internal/scripts/docker/README similarity index 100% rename from docker/README rename to internal/scripts/docker/README diff --git a/docker/check_env.sh b/internal/scripts/docker/check_env.sh similarity index 100% rename from docker/check_env.sh rename to internal/scripts/docker/check_env.sh diff --git a/docker/docker-clean b/internal/scripts/docker/docker-clean similarity index 100% rename from docker/docker-clean rename to internal/scripts/docker/docker-clean diff --git a/docker/docker-compose.yml b/internal/scripts/docker/docker-compose.yml similarity index 100% rename from docker/docker-compose.yml rename to internal/scripts/docker/docker-compose.yml diff --git a/docker/docker-entrypoint.sh b/internal/scripts/docker/docker-entrypoint.sh similarity index 100% rename from docker/docker-entrypoint.sh rename to internal/scripts/docker/docker-entrypoint.sh diff --git a/docker/fix-permissions.sh b/internal/scripts/docker/fix-permissions.sh similarity index 100% rename from docker/fix-permissions.sh rename to internal/scripts/docker/fix-permissions.sh diff --git a/internal/scripts/docker/hooks/build b/internal/scripts/docker/hooks/build new file mode 100644 index 00000000..53917e2b --- /dev/null +++ b/internal/scripts/docker/hooks/build @@ -0,0 +1,13 @@ +#!/bin/bash + +# TODO: Add logic for handling dependencies +# - develop or branches from develop should build against develop +# - main_vX.Y or branches from main_vX.Y should build against main_v{X-3}.Y +# - vX.Y.Z tags should build against v{X-3}.Y.? tags + +docker build -t $IMAGE_NAME \ +--build-arg METVIEWER_GIT_NAME=$SOURCE_BRANCH \ +--build-arg METCALCPY_GIT_NAME=develop \ +--build-arg METDATAIO_GIT_NAME=develop \ +--build-arg METPLOTPY_GIT_NAME=develop . + diff --git a/docker/init_singularity.sh b/internal/scripts/docker/init_singularity.sh similarity index 100% rename from docker/init_singularity.sh rename to internal/scripts/docker/init_singularity.sh diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties new file mode 100644 index 00000000..3586fc31 --- /dev/null +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -0,0 +1,13 @@ +# Project and source code settings +sonar.projectKey=METviewer +sonar.projectName=METviewer +sonar.projectVersion=SONAR_PROJECT_VERSION +sonar.branch.name=SONAR_BRANCH_NAME +sonar.sources=java +sonar.java.libraries=dist/lib +sonar.java.binaries=dist/metviewer/WEB-INF/classes +sonar.sourceEncoding=UTF-8 + +# SonarQube server +sonar.host.url=SONAR_HOST_URL +sonar.token=SONAR_TOKEN