Feature #2379 main_v11.1 sonarqube GHA (#2848)

* Per #2379, migrating largely the same changes for #2379 into the main_v11.1 branch. The difference is that --enable-all is not used since that is not a valid configuration option for MET version 11.1.0.

* Hotfix related to #2379. The sonar.newCode.referenceBranch and sonar.branch.name cannot be set to the same string! Only add the newCode definition when they differ.

.github/jobs/build_sonarqube_image.sh

@@ -0,0 +1,45 @@
+#! /bin/bash
+
+source ${GITHUB_WORKSPACE}/.github/jobs/bash_functions.sh
+
+DOCKERHUB_TAG=met-sonarqube-gha
+
+DOCKERFILE_PATH=${GITHUB_WORKSPACE}/internal/scripts/docker/Dockerfile.sonarqube
+
+CMD_LOGFILE=${GITHUB_WORKSPACE}/sonarqube_build.log
+
+#
+# Define the $SONAR_REFERENCE_BRANCH as the
+#   - Target of any requests
+#   - Manual setting for workflow dispatch
+#   - Source branch for any pushes (e.g. develop)
+#
+if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+  export SONAR_REFERENCE_BRANCH=${GITHUB_BASE_REF}
+elif [ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]; then
+  export SONAR_REFERENCE_BRANCH=${WD_REFERENCE_BRANCH}
+else
+  export SONAR_REFERENCE_BRANCH=${SOURCE_BRANCH}
+fi
+
+echo SONAR_REFERENCE_BRANCH=${SONAR_REFERENCE_BRANCH}
+
+time_command docker build -t ${DOCKERHUB_TAG} \
+    --build-arg MET_BASE_REPO \
+    --build-arg MET_BASE_TAG \
+    --build-arg SOURCE_BRANCH \
+    --build-arg SONAR_SCANNER_VERSION \
+    --build-arg SONAR_HOST_URL \
+    --build-arg SONAR_TOKEN \
+    --build-arg SONAR_REFERENCE_BRANCH \
+    -f $DOCKERFILE_PATH ${GITHUB_WORKSPACE}
+if [ $? != 0 ]; then
+  cat ${CMD_LOGFILE}
+  exit 1
+fi
+
+# Copy the .scannerwork directory from the image
+id=$(docker create ${DOCKERHUB_TAG})
+time_command docker cp $id:/met/.scannerwork /tmp/met_scannerwork 
+docker rm -v $id
+

.github/pull_request_template.md

@@ -22,6 +22,9 @@ If **yes**, describe the new output and/or changes to the existing output:</br>
 - [ ] Will this PR result in changes to existing METplus Use Cases? **[Yes or No]**</br>
 If **yes**, create a new **Update Truth** [METplus issue](https://github.com/dtcenter/METplus/issues/new/choose) to describe them.
 
+- [ ] Do these changes introduce new SonarQube findings? **[Yes or No]**</br>
+If **yes**, please describe:
+
 - [ ] Please complete this pull request review by **[Fill in date]**.</br>
 
 ## Pull Request Checklist ##

.github/workflows/sonarqube.yml

@@ -0,0 +1,102 @@
+name: SonarQube Scan
+
+# Run SonarQube for Pull Requests and changes to the develop and main_vX.Y branches
+
+on:
+
+  # Trigger analysis for pushes to develop and main_vX.Y branches
+  push:
+    branches:
+      - develop
+      - 'main_v**'
+    paths-ignore:
+      - 'docs/**'
+      - '.github/pull_request_template.md'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '.github/labels/**'
+      - '**/README.md'
+      - '**/LICENSE.md'
+
+  # Trigger analysis for pull requests to develop and main_vX.Y branches
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches:
+      - develop
+      - 'main_v**'
+    paths-ignore:
+      - 'docs/**'
+      - '.github/pull_request_template.md'
+      - '.github/ISSUE_TEMPLATE/**'
+      - '.github/labels/**'
+      - '**/README.md'
+      - '**/LICENSE.md'
+
+  workflow_dispatch:
+    inputs:
+      reference_branch:
+        description: 'Reference Branch'
+        default: develop 
+        type: string
+
+jobs:
+  build:
+    name: SonarQube Scan 
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - uses: actions/checkout@v4
+        with:
+          # Disable shallow clones for better analysis
+          fetch-depth: 0
+
+      - name: Create output directories
+        run: mkdir -p ${RUNNER_WORKSPACE}/logs
+
+      - name: Get branch name
+        id: get_branch_name
+        run: echo branch_name=${GITHUB_REF#refs/heads/} >> $GITHUB_OUTPUT
+
+      - name: SonarQube Scan in Docker
+        run: .github/jobs/build_sonarqube_image.sh
+        env:
+          MET_BASE_REPO: met-base
+          MET_BASE_TAG: v3.2
+          SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}
+          WD_REFERENCE_BRANCH: ${{ github.event.inputs.reference_branch }}
+          SONAR_SCANNER_VERSION: 5.0.1.3006
+          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+      - name: SonarQube Python Quality Gate check
+        id: sonarqube-python-quality-gate-check
+        uses: sonarsource/sonarqube-quality-gate-action@master
+        with:
+          scanMetadataReportFile: /tmp/met_scannerwork/python-report-task.txt
+        timeout-minutes: 5
+        env:
+          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+      - name: SonarQube CXX Quality Gate check
+        id: sonarqube-cxx-quality-gate-check
+        uses: sonarsource/sonarqube-quality-gate-action@master
+        with:
+          scanMetadataReportFile: /tmp/met_scannerwork/cxx-report-task.txt
+        timeout-minutes: 5
+        env:
+          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+      - name: Copy log files into logs directory
+        if: always()
+        run: cp ${GITHUB_WORKSPACE}/*.log ${RUNNER_WORKSPACE}/logs/
+
+      - name: Upload logs as artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs_sonarqube
+          path: ${{ runner.workspace }}/logs
+          if-no-files-found: ignore
+

internal/scripts/docker/Dockerfile.sonarqube

@@ -0,0 +1,95 @@
+ARG MET_BASE_REPO=met-base
+ARG MET_BASE_TAG=v3.2
+
+FROM dtcenter/${MET_BASE_REPO}:${MET_BASE_TAG}
+MAINTAINER John Halley Gotway <[email protected]>
+
+#
+# This Dockerfile checks out MET from GitHub and runs the
+# SonarQube static code analysis on the specified branch or tag.
+# https://docs.sonarqube.org/latest/analysis/scan/sonarscanner/
+#
+ARG SONAR_SCANNER_VERSION=5.0.1.3006
+ARG SONAR_HOST_URL
+ARG SONAR_TOKEN
+ARG SOURCE_BRANCH
+ARG SONAR_REFERENCE_BRANCH
+
+#
+# SONAR_HOST_URL is required.
+#
+RUN if [ "x${SONAR_HOST_URL}" = "x" ]; then \
+      echo "ERROR: SONAR_HOST_URL undefined! Rebuild with \"--build-arg SONAR_HOST_URL={url}\""; \
+      exit 1; \
+    fi
+
+#
+# SONAR_TOKEN is required.
+#
+RUN if [ "x${SONAR_TOKEN}" = "x" ]; then \
+      echo "ERROR: SONAR_TOKEN undefined! Rebuild with \"--build-arg SONAR_TOKEN={token}\""; \
+      exit 1; \
+    fi
+
+#
+# SOURCE_BRANCH is the branch name of the MET source code.
+#
+RUN if [ "x${SOURCE_BRANCH}" = "x" ]; then \
+      echo "ERROR: SOURCE_BRANCH undefined! Rebuild with \"--build-arg SOURCE_BRANCH={branch name}\""; \
+      exit 1; \
+    else \
+      echo "Build Argument SOURCE_BRANCH=${SOURCE_BRANCH}"; \
+    fi
+
+#
+# SONAR_REFERENCE_BRANCH defines to the version against which this scan should be compared.
+#
+RUN if [ "x${SONAR_REFERENCE_BRANCH}" = "x" ]; then \
+      echo "ERROR: SONAR_REFERENCE_BRANCH undefined! Rebuild with \"--build-arg SONAR_REFERENCE_BRANCH={branch name}\""; \
+      exit 1; \
+    else \
+      echo "Build Argument SONAR_REFERENCE_BRANCH=${SONAR_REFERENCE_BRANCH}"; \
+    fi
+
+ENV MET_GIT_NAME ${SOURCE_BRANCH}
+ENV MET_REPO_DIR /met/MET-${MET_GIT_NAME}
+ENV MET_GIT_URL  https://github.com/dtcenter/MET
+
+#
+# Download and install the Sonar software.
+#
+RUN echo "Installing SonarQube into $HOME/.sonar" \
+ && mkdir -p $HOME/.sonar \
+ && curl -sSLo $HOME/.sonar/sonar-scanner.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux.zip \
+ && unzip -o $HOME/.sonar/sonar-scanner.zip -d $HOME/.sonar/ \
+ && echo export PATH="$HOME/.sonar/sonar-scanner-${SONAR_SCANNER_VERSION}-linux/bin:\$PATH" >> $HOME/.bashrc \
+ && curl -sSLo $HOME/.sonar/build-wrapper-linux-x86.zip ${SONAR_HOST_URL}/static/cpp/build-wrapper-linux-x86.zip \
+ && unzip -o $HOME/.sonar/build-wrapper-linux-x86.zip -d $HOME/.sonar/ \
+ && echo export PATH="$HOME/.sonar/build-wrapper-linux-x86:\$PATH" >> $HOME/.bashrc
+
+#
+# Update the OS, as needed.
+#
+RUN apt update
+
+#
+# Set the working directory.
+#
+WORKDIR /met
+
+#
+# Copy MET Download and install MET.
+#
+RUN echo "Copying MET into ${MET_REPO_DIR}" \
+ && mkdir -p ${MET_REPO_DIR}
+
+COPY . ${MET_REPO_DIR}
+
+RUN if [ ! -e "${MET_REPO_DIR}/configure.ac" ]; then \
+    echo "ERROR: docker build must be run from the MET directory: `ls`"; \
+    exit 1; \
+  fi
+
+RUN cd ${MET_REPO_DIR} \
+ && internal/scripts/docker/build_met_sonarqube.sh
+

internal/scripts/docker/build_met_sonarqube.sh

@@ -0,0 +1,142 @@
+#!/bin/bash
+#
+# Run SonarQube Source Code Analyzer within a Docker container
+#=======================================================================
+#
+# This build_met_sonarqube.sh script must be run from the top-level
+# directory of the MET repository to be analyzed. It runs SonarQube to
+# scan both the Python and C/C++ MET source code.
+#
+# Usage: internal/scripts/docker/build_met_sonarqube.sh
+#
+# Required Enviornment Variables:
+#   SONAR_HOST_URL
+#   SONAR_TOKEN
+#   MET_GIT_NAME
+#   SONAR_REFERENCE_BRANCH
+#
+#=======================================================================
+
+# Check that this is being run from the top-level MET directory
+if [ ! -e internal/scripts/docker/build_met_sonarqube.sh ]; then
+  echo "ERROR: ${0} -> must be run from the top-level MET directory"
+  exit 1
+fi
+
+echo "Running script to scan MET with SonarQube in Docker"
+
+# Source the docker build environment
+source ~/.bashrc
+source internal/scripts/environment/development.docker 
+source .github/jobs/bash_functions.sh
+
+# Check required environment variables
+if [ -z ${SONAR_HOST_URL+x} ]; then
+  echo "ERROR: ${0} -> \$SONAR_HOST_URL not defined!"
+  exit 1
+fi
+if [ -z ${SONAR_TOKEN+x} ]; then
+  echo "ERROR: ${0} -> \$SONAR_TOKEN not defined!"
+  exit 1
+fi
+if [ -z ${MET_GIT_NAME+x} ]; then
+  echo "ERROR: ${0} -> \$MET_GIT_NAME not defined!"
+  exit 1
+fi
+if [ -z ${SONAR_REFERENCE_BRANCH+x} ]; then
+  echo "ERROR: ${0} -> \$SONAR_REFERENCE_BRANCH not defined!"
+  exit 1
+fi
+
+# Locate the wrapper
+WRAPPER_NAME=build-wrapper-linux-x86-64
+SONAR_WRAPPER=$(which $WRAPPER_NAME 2> /dev/null)
+
+if [ ! -e $SONAR_WRAPPER ]; then
+  echo "ERROR: ${0} -> $WRAPPER_NAME not found in the path"
+  exit 1
+else
+  echo "SONAR_WRAPPER=$SONAR_WRAPPER"
+fi
+
+# Locate the scanner
+SCANNER_NAME=sonar-scanner
+SONAR_SCANNER=$(which $SCANNER_NAME 2> /dev/null)
+
+if [ ! -e $SONAR_SCANNER ]; then
+  echo "ERROR: ${0} -> $SCANNER_NAME not found in the path"
+  exit 1
+else
+  echo "SONAR_SCANNER=$SONAR_SCANNER"
+fi
+
+# Set output directory name
+if [ -z ${SONARQUBE_OUT_DIR} ]; then
+  export SONARQUBE_OUT_DIR=bw-outputs
+fi
+
+# Store the full path to the scripts directory
+SONAR_PROPERTIES_DIR=internal/scripts/sonarqube
+SONAR_PROPERTIES=sonar-project.properties
+
+# Copy sonar-project.properties for Python code
+[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES
+sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \
+    -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \
+    -e "s|SONAR_PROJECT_KEY|MET-GHA-Python|" \
+    -e "s|SONAR_PROJECT_NAME|MET GHA Python|" \
+    -e "s|SONAR_BRANCH_NAME|$MET_GIT_NAME|" \
+    $SONAR_PROPERTIES_DIR/python.sonar-project.properties > $SONAR_PROPERTIES
+
+# The source and reference branches must differ to define new code
+if [ "$MET_GIT_NAME" != "$SONAR_REFERENCE_BRANCH" ]; then
+  echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES
+fi
+
+# Run SonarQube scan for Python code
+time_command $SONAR_SCANNER
+
+# Copy the Python scan report-task.txt file
+mkdir -p /met/.scannerwork
+cp .scannerwork/report-task.txt /met/.scannerwork/python-report-task.txt
+
+# Copy sonar-project.properties for C/C++ code
+[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES
+sed -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \
+    -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \
+    -e "s|SONAR_PROJECT_KEY|MET-GHA-CXX|" \
+    -e "s|SONAR_PROJECT_NAME|MET GHA CXX|" \
+    -e "s|SONAR_BRANCH_NAME|$MET_GIT_NAME|" \
+    $SONAR_PROPERTIES_DIR/sonar-project.properties > $SONAR_PROPERTIES
+
+# The source and reference branches must differ to define new code
+if [ "$MET_GIT_NAME" != "$SONAR_REFERENCE_BRANCH" ]; then
+  echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES
+fi
+
+# Run the configure script
+time_command ./configure \
+  BUFRLIB_NAME=${BUFRLIB_NAME} \
+  GRIB2CLIB_NAME=${GRIB2CLIB_NAME} \
+  --enable-grib2 \
+  --enable-modis \
+  --enable-mode_graphics \
+  --enable-lidar2nc \
+  --enable-python
+  CPPFLAGS="-I/usr/local/include -I/usr/local/include/freetype2 -I/usr/local/include/cairo" \
+  LIBS="-ltirpc"
+
+# Run make clean
+time_command make clean
+
+# Run SonarQube make
+time_command $SONAR_WRAPPER --out-dir $SONARQUBE_OUT_DIR make
+
+# Run SonarQube scan for C/C++ code
+time_command $SONAR_SCANNER
+
+# Copy the C/C++ scan report-task.txt file
+mkdir -p /met/.scannerwork
+cp .scannerwork/report-task.txt /met/.scannerwork/cxx-report-task.txt
+
+[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES