From 17932278f9c13600186610f5ef9763647b0e23b4 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 4 Apr 2024 08:52:21 -0600 Subject: [PATCH 1/2] Per #289, same set of changes from PR #290, but for the main_v2.1 branch instead. --- .github/jobs/configure_sonarqube.sh | 65 ++++++++++ .github/pull_request_template.md | 3 + .github/workflows/sonarqube.yml | 76 ++++++++++++ internal/scripts/sonarqube/run_sonarqube.sh | 34 ++++-- .../sonarqube/sonar-project.properties | 13 +- .../scanning/sonarqube/development.seneca | 10 -- .../scanning/sonarqube/run_nightly.sh | 78 ------------ .../scanning/sonarqube/run_sonarqube.sh | 115 ------------------ .../sonarqube/sonar-project.properties | 17 --- 9 files changed, 175 insertions(+), 236 deletions(-) create mode 100755 .github/jobs/configure_sonarqube.sh create mode 100644 .github/workflows/sonarqube.yml delete mode 100644 internal_tests/scanning/sonarqube/development.seneca delete mode 100755 internal_tests/scanning/sonarqube/run_nightly.sh delete mode 100755 internal_tests/scanning/sonarqube/run_sonarqube.sh delete mode 100644 internal_tests/scanning/sonarqube/sonar-project.properties diff --git a/.github/jobs/configure_sonarqube.sh b/.github/jobs/configure_sonarqube.sh new file mode 100755 index 00000000..929b03fa --- /dev/null +++ b/.github/jobs/configure_sonarqube.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Constants +SONAR_PROPERTIES_DIR=internal/scripts/sonarqube +SONAR_PROPERTIES=sonar-project.properties + +# Check that this is being run from the top-level METdataio directory +if [ ! -e $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES ]; then + echo "ERROR: ${0} -> must be run from the top-level METdataio directory" + exit 1 +fi + +# Check required environment variables +if [ -z ${SOURCE_BRANCH+x} ]; then + echo "ERROR: ${0} -> \$SOURCE_BRANCH not defined!" + exit 1 +fi +if [ -z ${WD_REFERENCE_BRANCH+x} ]; then + echo "ERROR: ${0} -> \$WD_REFERENCE_BRANCH not defined!" + exit 1 +fi +if [ -z ${SONAR_HOST_URL+x} ]; then + echo "ERROR: ${0} -> \$SONAR_HOST_URL not defined!" + exit 1 +fi +if [ -z ${SONAR_TOKEN+x} ]; then + echo "ERROR: ${0} -> \$SONAR_TOKEN not defined!" + exit 1 +fi + +# Define the version string +SONAR_PROJECT_VERSION=$(cat docs/version | cut -d'=' -f2 | tr -d '" ') + +# +# Define the $SONAR_REFERENCE_BRANCH as the +# - Target of any requests +# - Manual setting for workflow dispatch +# - Source branch for any pushes (e.g. develop) +# +if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then + export SONAR_REFERENCE_BRANCH=$GITHUB_BASE_REF +elif [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then + export SONAR_REFERENCE_BRANCH=$WD_REFERENCE_BRANCH +else + export SONAR_REFERENCE_BRANCH=$SOURCE_BRANCH +fi + +# Configure the sonar-project.properties +[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES +sed -e "s|SONAR_PROJECT_KEY|METdataio-GHA|" \ + -e "s|SONAR_PROJECT_NAME|METdataio GHA|" \ + -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_BRANCH_NAME|$SOURCE_BRANCH|" \ + $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES + +# Define new code when the source and reference branches differ +if [ "$SOURCE_BRANCH" != "$SONAR_REFERENCE_BRANCH" ]; then + echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES +fi + +echo "Contents of the $SONAR_PROPERTIES file:" +cat $SONAR_PROPERTIES + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index dda54835..342abeb9 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,6 +11,9 @@ - [ ] Will this PR result in changes to the test suite? **[Yes or No]**
If **yes**, describe the new output and/or changes to the existing output:
+- [ ] Do these changes introduce new SonarQube findings? **[Yes or No]**
+If **yes**, please describe: + - [ ] Please complete this pull request review by **[Fill in date]**.
## Pull Request Checklist ## diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml new file mode 100644 index 00000000..d48e1237 --- /dev/null +++ b/.github/workflows/sonarqube.yml @@ -0,0 +1,76 @@ +name: SonarQube Scan + +# Run SonarQube for Pull Requests and changes to the develop and main_vX.Y branches + +on: + + # Trigger analysis for pushes to develop and main_vX.Y branches + push: + branches: + - develop + - 'main_v**' + paths-ignore: + - 'docs/**' + - '.github/pull_request_template.md' + - '.github/ISSUE_TEMPLATE/**' + - '**/README.md' + - '**/LICENSE.md' + + # Trigger analysis for pull requests to develop and main_vX.Y branches + pull_request: + types: [opened, synchronize, reopened] + branches: + - develop + - 'main_v**' + paths-ignore: + - 'docs/**' + - '.github/pull_request_template.md' + - '.github/ISSUE_TEMPLATE/**' + - '**/README.md' + - '**/LICENSE.md' + + workflow_dispatch: + inputs: + reference_branch: + description: 'Reference Branch' + default: develop + type: string + +jobs: + sonarqube: + name: SonarQube Scan + runs-on: ubuntu-latest + + steps: + + - uses: actions/checkout@v4 + with: + # Disable shallow clones for better analysis + fetch-depth: 0 + + - name: Get branch name + id: get_branch_name + run: echo branch_name=${GITHUB_REF#refs/heads/} >> $GITHUB_OUTPUT + + - name: Configure SonarQube + run: .github/jobs/configure_sonarqube.sh + env: + SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }} + WD_REFERENCE_BRANCH: ${{ github.event.inputs.reference_branch }} + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: SonarQube Scan + uses: sonarsource/sonarqube-scan-action@master + env: + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: SonarQube Quality Gate check + id: sonarqube-quality-gate-check + uses: sonarsource/sonarqube-quality-gate-action@master + # Force to fail step after specific time. + timeout-minutes: 5 + env: + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/internal/scripts/sonarqube/run_sonarqube.sh b/internal/scripts/sonarqube/run_sonarqube.sh index 85f30833..49fea266 100755 --- a/internal/scripts/sonarqube/run_sonarqube.sh +++ b/internal/scripts/sonarqube/run_sonarqube.sh @@ -1,10 +1,10 @@ #!/bin/bash # -# Run SonarQube Source Code Analyzer on a specified revision of MET +# Run SonarQube Source Code Analyzer for METdataio #======================================================================= # # This run_sonarqube.sh script will check out the specified version -# of MET and run the SonarQube Source Code Analyzer on it. First, +# of METdataio and run the SonarQube Source Code Analyzer on it. First, # go to the directory where you would like the SCA output written and # then run: # @@ -12,9 +12,9 @@ # METdataio/sonarqube/run_sonarqube.sh name # # Usage: run_sonarqube.sh name -# Test the specified branched version of MET: +# Test the specified branched version of METdataio: # run_sonarqube.sh {branch name} -# Test the specified tagged version of MET: +# Test the specified tagged version of METdataio: # run_sonarqube.sh {tag name} # #======================================================================= @@ -33,6 +33,16 @@ function usage { # Check for arguments if [[ $# -lt 1 ]]; then usage; exit; fi +# Check that SONAR_TOKEN and SONAR_HOST_URL are defined +if [ -z ${SONAR_TOKEN} ]; then + echo "ERROR: SONAR_TOKEN must be set" + exit 1 +fi +if [ -z ${SONAR_HOST_URL} ]; then + echo "ERROR: SONAR_HOST_URL must be set" + exit 1 +fi + # Check that SONARQUBE_WRAPPER_BIN is defined if [ -z ${SONARQUBE_WRAPPER_BIN} ]; then which build-wrapper-linux-x86-64 2> /dev/null @@ -87,7 +97,6 @@ function run_command() { return ${STATUS} } - # Store the full path to the scripts directory SCRIPT_DIR=`dirname $0` if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi @@ -102,14 +111,21 @@ run_command "git clone ${GIT_REPO} ${REPO_DIR}" run_command "cd ${REPO_DIR}" run_command "git checkout ${1}" +# Define the version string +SONAR_PROJECT_VERSION=$(cat docs/version | cut -d'=' -f2 | tr -d '" ') + SONAR_PROPERTIES=sonar-project.properties -# Copy sonar-project.properties for Python code +# Configure the sonar-project.properties [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES -cp -p $SCRIPT_DIR/sonar-project.properties $SONAR_PROPERTIES +sed -e "s|SONAR_PROJECT_KEY|METdataio_NB|" \ + -e "s|SONAR_PROJECT_NAME|METdataio Nightly Build|" \ + -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_BRANCH_NAME|${1}|" \ + $SCRIPT_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES # Run SonarQube scan for Python code run_command "${SONARQUBE_SCANNER_BIN}/sonar-scanner" -# Run SonarQube report generator to make a PDF file -#TODAY=`date +%Y%m%d` diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index 7462a32e..ad9a6f49 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -1,6 +1,6 @@ -sonar.projectKey=org.sonarqube:METdataio_NB -sonar.projectName=METdataio Nightly Build -sonar.projectVersion=1.0 +sonar.projectKey=SONAR_PROJECT_KEY +sonar.projectName=SONAR_PROJECT_NAME +sonar.projectVersion=SONAR_PROJECT_VERSION sonar.sources=METdbLoad,METreadnc @@ -10,8 +10,7 @@ sonar.sources=METdbLoad,METreadnc sonar.sourceEncoding=UTF-8 #----- Default SonarQube server -#sonar.host.url=http://localhost:9000 -sonar.host.url=http://mandan:9000 +sonar.host.url=SONAR_HOST_URL -sonar.login=met -sonar.password=met@sonar.ucar +sonar.token=SONAR_TOKEN +sonar.branch.name=SONAR_BRANCH_NAME diff --git a/internal_tests/scanning/sonarqube/development.seneca b/internal_tests/scanning/sonarqube/development.seneca deleted file mode 100644 index 160b3dcf..00000000 --- a/internal_tests/scanning/sonarqube/development.seneca +++ /dev/null @@ -1,10 +0,0 @@ -# Define the development environment for NCAR project machine seneca -# Based on settings in /usr/local/src/met/README.snat - -# Top-level MET project directory -MET_PROJ_DIR=/d1/projects/MET - -# SonarQube -export SONARQUBE_DIR=/d1/projects/SonarQube/ -export SONARQUBE_WRAPPER_BIN=$SONARQUBE_DIR/build-wrapper-linux-x86 -export SONARQUBE_SCANNER_BIN=$SONARQUBE_DIR/sonar-scanner-4.6.2.2472-linux/bin diff --git a/internal_tests/scanning/sonarqube/run_nightly.sh b/internal_tests/scanning/sonarqube/run_nightly.sh deleted file mode 100755 index 78bdaa0f..00000000 --- a/internal_tests/scanning/sonarqube/run_nightly.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -# -# Run nightly SonarQube scan -#======================================================================= -# -# This run_nightly.sh script calls the run_sonarqube.sh script. -# It is intented to be run nightly through cron. Output should be -# directed to the LOGFILE, per cron convention. To run this script, use -# the following commands: -# -# git clone https://github.com/dtcenter/METdataio -# METdataio/sonarqube/run_nightly.sh name -# -# Usage: run_nightly.sh name -# where "name" speci1fies a branch, tag, or hash -# -# For example, scan the develop branch: -# run_nightly.sh develop -# -#======================================================================= - -# Constants -EMAIL_LIST="johnhg@ucar.edu hsoh@ucar.edu mccabe@ucar.edu fisherh@ucar.edu" -TEST_EMAIL_LIST="hsoh@ucar.edu" -KEEP_DAYS=5 -GIT_REPO_NAME=METdataio - -function usage { - echo - echo "USAGE: run_nightly.sh name" - echo " where \"name\" specifies a branch, tag, or hash." - echo -} - -# Check for arguments -if [ $# -lt 1 ]; then usage; exit 1; fi - -# Store the full path to the scripts directory -SCRIPT_DIR=`dirname $0` -if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi - -# Define the development environment -ENV_FILE=${SCRIPT_DIR}/development.`hostname` -if [[ ! -e ${ENV_FILE} ]]; then - echo "$0: ERROR -> Development environment file missing: ${ENV_FILE}" - exit 1 -fi -source ${ENV_FILE} - -SONARQUBE_WORK_DIR=${MET_PROJ_DIR}/MET_regression/sonarqube_${GIT_REPO_NAME} - -# Delete old directories -find ${SONARQUBE_WORK_DIR} -mtime +${KEEP_DAYS} -name "NB*" | \ - xargs rm -rf - -# Create and switch to a run directory -TODAY=`date +%Y%m%d` -YESTERDAY=`date -d "1 day ago" +%Y%m%d` -RUN_DIR=${SONARQUBE_WORK_DIR}/NB${TODAY} -[[ -e ${RUN_DIR} ]] && rm -rf ${RUN_DIR} -mkdir -p ${RUN_DIR} -cd ${RUN_DIR} - -# Create a logfile -LOGFILE=${RUN_DIR}/run_sonarqube_${TODAY}.log - -# Run scan and check for bad return status -${SCRIPT_DIR}/run_sonarqube.sh ${1} >& ${LOGFILE} -if [[ $? -ne 0 ]]; then - [ ! -z $2 ] && EMAIL_LIST=$TEST_EMAIL_LIST - echo "$0: The nightly SonarQube scanning for $GIT_REPO_NAME FAILED in `basename ${RUN_DIR}`." >> ${LOGFILE} - cat ${LOGFILE} | mail -s "$GIT_REPO_NAME SonarQube scanning failed for ${1} in `basename ${RUN_DIR}` (autogen msg)" ${EMAIL_LIST} - exit 1 -fi - -# Convert SonarQube report from pdf to html - -exit 0 diff --git a/internal_tests/scanning/sonarqube/run_sonarqube.sh b/internal_tests/scanning/sonarqube/run_sonarqube.sh deleted file mode 100755 index 85f30833..00000000 --- a/internal_tests/scanning/sonarqube/run_sonarqube.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# -# Run SonarQube Source Code Analyzer on a specified revision of MET -#======================================================================= -# -# This run_sonarqube.sh script will check out the specified version -# of MET and run the SonarQube Source Code Analyzer on it. First, -# go to the directory where you would like the SCA output written and -# then run: -# -# git clone https://github.com/dtcenter/METdataio -# METdataio/sonarqube/run_sonarqube.sh name -# -# Usage: run_sonarqube.sh name -# Test the specified branched version of MET: -# run_sonarqube.sh {branch name} -# Test the specified tagged version of MET: -# run_sonarqube.sh {tag name} -# -#======================================================================= - -# Constants -GIT_REPO_NAME=METdataio -GIT_REPO="https://github.com/dtcenter/${GIT_REPO_NAME}" - -function usage { - echo - echo "USAGE: $(basename $0) name" - echo " where \"name\" specifies a branch, tag, or hash." - echo -} - -# Check for arguments -if [[ $# -lt 1 ]]; then usage; exit; fi - -# Check that SONARQUBE_WRAPPER_BIN is defined -if [ -z ${SONARQUBE_WRAPPER_BIN} ]; then - which build-wrapper-linux-x86-64 2> /dev/null - if [ $? -eq 0 ]; then - SONARQUBE_WRAPPER_BIN=$(which build-wrapper-linux-x86-64 2> /dev/null) - else - which build-wrapper 2> /dev/null - if [ $? -eq 0 ]; then - SONARQUBE_WRAPPER_BIN=$(which build-wrapper 2> /dev/null) - else - echo "ERROR: SONARQUBE_WRAPPER_BIN must be set" - exit 1 - fi - fi -fi -if [ ! -e ${SONARQUBE_WRAPPER_BIN} ]; then - echo "ERROR: SONARQUBE_WRAPPER_BIN (${SONARQUBE_WRAPPER_BIN}) does not exist" - exit 1 -fi - -# Check that SONARQUBE_SCANNER_BIN is defined -if [ -z ${SONARQUBE_SCANNER_BIN} ]; then - which sonar-scanner 2> /dev/null - if [ $? -eq 0 ]; then - SONARQUBE_SCANNER_BIN=$(which sonar-scanner 2> /dev/null) - else - echo "ERROR: SONARQUBE_SCANNER_BIN must be set" - exit 1 - fi -fi -if [ ! -e ${SONARQUBE_SCANNER_BIN} ]; then - echo "ERROR: SONARQUBE_SCANNER_BIN (${SONARQUBE_SCANNER_BIN}) does not exist" - exit 1 -fi - -# Sub-routine for running a command and checking return status -function run_command() { - - # Print the command being called - echo "CALLING: $1" - - # Run the command and store the return status - $1 - STATUS=$? - - # Check return status - if [[ ${STATUS} -ne 0 ]]; then - echo "ERROR: Command returned with non-zero status ($STATUS): $1" - exit ${STATUS} - fi - - return ${STATUS} -} - - -# Store the full path to the scripts directory -SCRIPT_DIR=`dirname $0` -if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi - -# Clone repo into a sub-directory and checkout the requested version -REPO_DIR="${GIT_REPO_NAME}-${1}" - -if [ -e ${REPO_DIR} ]; then - run_command "rm -rf ${REPO_DIR}" -fi -run_command "git clone ${GIT_REPO} ${REPO_DIR}" -run_command "cd ${REPO_DIR}" -run_command "git checkout ${1}" - -SONAR_PROPERTIES=sonar-project.properties - -# Copy sonar-project.properties for Python code -[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES -cp -p $SCRIPT_DIR/sonar-project.properties $SONAR_PROPERTIES - -# Run SonarQube scan for Python code -run_command "${SONARQUBE_SCANNER_BIN}/sonar-scanner" - -# Run SonarQube report generator to make a PDF file -#TODAY=`date +%Y%m%d` diff --git a/internal_tests/scanning/sonarqube/sonar-project.properties b/internal_tests/scanning/sonarqube/sonar-project.properties deleted file mode 100644 index 7462a32e..00000000 --- a/internal_tests/scanning/sonarqube/sonar-project.properties +++ /dev/null @@ -1,17 +0,0 @@ -sonar.projectKey=org.sonarqube:METdataio_NB -sonar.projectName=METdataio Nightly Build -sonar.projectVersion=1.0 - -sonar.sources=METdbLoad,METreadnc - -# The build-wrapper output dir - -# Encoding of the source files -sonar.sourceEncoding=UTF-8 - -#----- Default SonarQube server -#sonar.host.url=http://localhost:9000 -sonar.host.url=http://mandan:9000 - -sonar.login=met -sonar.password=met@sonar.ucar From c7ded1e664178707cb941982c78d3460f2c40d2a Mon Sep 17 00:00:00 2001 From: bikegeek <3753118+bikegeek@users.noreply.github.com> Date: Thu, 4 Apr 2024 09:36:34 -0600 Subject: [PATCH 2/2] Update sonar-project.properties include checks for METreformat code --- internal/scripts/sonarqube/sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index ad9a6f49..8e3c4d05 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -2,7 +2,7 @@ sonar.projectKey=SONAR_PROJECT_KEY sonar.projectName=SONAR_PROJECT_NAME sonar.projectVersion=SONAR_PROJECT_VERSION -sonar.sources=METdbLoad,METreadnc +sonar.sources=METdbLoad,METreadnc,METreformat # The build-wrapper output dir