diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..f2abc025c --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,38 @@ +name: "CodeQL Advanced" + +on: + push: + pull_request: + schedule: + - cron: '25 22 * * 0' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: 'ubuntu-latest' + permissions: + security-events: write + packages: read + + strategy: + fail-fast: false + matrix: + include: + - language: java-kotlin + - language: javascript-typescript + - language: python + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: none + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/sonar-qube.yml b/.github/workflows/sonar-qube.yml new file mode 100644 index 000000000..8d8dba203 --- /dev/null +++ b/.github/workflows/sonar-qube.yml @@ -0,0 +1,79 @@ +name: SonarQube Analysis + +on: + push: + pull_request: + +env: + java-version: '11' + gradle-version: '8.0.2' + +jobs: + sonar: + name: Run SonarQube Analysis + runs-on: ubuntu-latest + + services: + sonarqube: + image: sonarqube:community + ports: + - 9000:9000 + options: >- + --health-cmd="curl -s -u admin:admin http://localhost:9000/api/system/health | grep -o GREEN" + --health-interval=10s + --health-timeout=10s + --health-retries=60 + env: + SONAR_ES_BOOTSTRAP_CHECKS_DISABLE: "true" + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v4 + with: + distribution: 'corretto' + java-version: | + 11 + - name: Cache SonarQube Scanner + uses: actions/cache@v3 + with: + path: ~/.sonar/cache + key: sonar-cache + restore-keys: sonar-cache + - uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: 8.0.2 + gradle-home-cache-cleanup: true + - name: Run Gradle Build + run: ./gradlew copyDependencies + - name: Run SonarQube Scanner + run: | + curl -sLo sonar-scanner-cli.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.2.1.4610-linux-x64.zip + unzip -q sonar-scanner-cli.zip -d $HOME + export PATH="$HOME/sonar-scanner-6.2.1.4610-linux-x64/bin:$PATH" + sonar-scanner \ + -Dsonar.projectKey=local_project \ + -Dsonar.sources=. \ + -Dsonar.host.url=http://localhost:9000 \ + -Dsonar.login=admin \ + -Dsonar.password=admin + - name: Wait for issues to be processed + run: sleep 60 + - name: Collect issues from the server + run: | + curl -s -u admin:admin "http://localhost:9000/api/issues/search?componentKeys=local_project" -o issues.json + echo "SonarQube Issues Cmdline Report:" + jq -r '.issues[] | "File: \(.component):\(.line), Rule: \(.rule), Message: \(.message)"' issues.json | sort + ISSUE_COUNT=$(jq '.issues | length' issues.json) + + if [ "$ISSUE_COUNT" -gt 0 ]; then + echo "❌ Build failed: Found $ISSUE_COUNT issues." + exit 1 + else + echo "✅ No issues found." + fi + - name: Upload SonarQube Artifacts + if: always() + uses: actions/upload-artifact@v3 + with: + name: sonar-reports + path: issues.json diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 000000000..1642d5ff1 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,267 @@ +# NOTE: +# - Do not include sonar.projectVersion or sonar.projectName as these are set automatically by the pipeline +# - Customize sonar.sources, sonar.exclusions, sonar.coverage.exclusions, sonar.tests and sonar +# unit test coverage reports based on your project + +# Refer to https://docs.sonarqube.org/latest/project-administration/narrowing-the-focus/ +# for details on sources and exclusions. Note also .gitignore which is used by sonarqube +# +sonar.sources=source, deployment + +# .git/ not available from codecommit +sonar.scm.disabled = true + +# Focusing sonarqube analysis on non test code first and reducing noise from analysis of test code. Projects +# can customize the exclusions to include analyzing of test code if desired +# Removed protobuf generated files +# Remove testing packages from code coverage +# Remove xml files +# Removed experimental features (migrationConsoleApi, dashboardsSanitizer) +sonar.exclusions=\ + **/build/**, \ + **/buildSrc/**, \ + **/*Test*/**, \ + **/*test*/**, \ + **/node_modules/**, \ + **/cdk.out/**, \ + **/config/**, \ + **/*.xml, \ + **/*.config, \ + **/migrationConsole/console_api/**, \ + **/dashboardsSanitizer/**, \ + **/DocumentsFromSnapshotMigration/docker/**, \ + **/jenkinsdocker/**, \ + **/__init__.py, \ + **/integ_test/**, \ + **/setup.py, \ + **/jest.config.js, \ + **/coverage/**/* + +# Code coverage Specific Properties +# sonar.coverage.exclusions= + +# For Java tests +sonar.java.source=11 +sonar.java.binaries=. +sonar.java.libraries= **/build/dependencies + +# For Python tests +sonar.python.version = 3.11 + +# For JavaScript/TypeScript tests +sonar.javascript.lcov.reportPaths= **/lcov.info + +# Encoding of the source files +sonar.sourceEncoding=UTF-8 + +# Rule specific exclusions +sonar.issue.ignore.multicriteria = \ + p1, \ + ts1, ts2, ts3, ts4, ts5, ts6, ts7, ts8, ts9, ts10, \ + j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, \ + autoclose, \ + comp1, comp2, comp3, comp4, \ + loop1, loop2, loop3, loop4, loop5, \ + f1, f2, f3, f4, f5, f6, f7, f8, \ + d1 + +sonar.issue.ignore.multicriteria.p1.ruleKey = python:S1135 +sonar.issue.ignore.multicriteria.p1.resourceKey = **/*.py + + + +sonar.issue.ignore.multicriteria.ts1.ruleKey = typescript:S1135 +sonar.issue.ignore.multicriteria.ts1.resourceKey = **/*.ts + +# CDK typescript requires resources to be created which aren't actually used in code but will be used in CFN template generation +sonar.issue.ignore.multicriteria.ts2.ruleKey = typescript:S1848 +sonar.issue.ignore.multicriteria.ts2.resourceKey = **/*.ts + +# We use an empty class (ClusterNoAuth) to receive a configuration option from the CDK Context +sonar.issue.ignore.multicriteria.ts3.ruleKey = typescript:S2094 +sonar.issue.ignore.multicriteria.ts3.resourceKey = **/common-utilities.ts + +# Disable complexity check for our CDK code; we know it's rough and a refactor will be risky and time consuming +# without an obvious payoff given its slow rate of change and our intention to replace it entirely +sonar.issue.ignore.multicriteria.ts4.ruleKey = typescript:S3776 +sonar.issue.ignore.multicriteria.ts4.resourceKey = **/cdk/opensearch-service-migration/**/*.ts + +# Ignore a hardcoded password for an OpenSearch stack used soley for integ testing purposes +# and called out as such. +sonar.issue.ignore.multicriteria.ts5.ruleKey = typescript:S2068 +sonar.issue.ignore.multicriteria.ts5.resourceKey = **/cdk/opensearch-service-migration/**/opensearch-container-stack.ts + +# Allow usage of the /tmp directory. This directory is used to temporarily download the snapshot +# into the ephemeral storage of a Fargate container with a single process running on it and a +# limited lifespan; seems safe to me. +sonar.issue.ignore.multicriteria.ts6.ruleKey = typescript:S5443 +sonar.issue.ignore.multicriteria.ts6.resourceKey = **/cdk/opensearch-service-migration/**/reindex-from-snapshot-stack.ts + +# Allow usage of the /tmp directory to store logs inside of a Kafka container +sonar.issue.ignore.multicriteria.ts7.ruleKey = typescript:S5443 +sonar.issue.ignore.multicriteria.ts7.resourceKey = **/cdk/opensearch-service-migration/**/kafka-stack.ts + +# Don't require versioning on a bucket that doesn't need it (the RFS SNapshot bucket) +sonar.issue.ignore.multicriteria.ts8.ruleKey = typescript:S6252 +sonar.issue.ignore.multicriteria.ts8.resourceKey = **/cdk/opensearch-service-migration/**/migration-assistance-stack.ts + +# Allow users to make the RFS Snapshot bucket public, even though it is private by default. +# Enables them to write to the bucket from anywhere (e.g. on-prem). +sonar.issue.ignore.multicriteria.ts9.ruleKey = typescript:S6281 +sonar.issue.ignore.multicriteria.ts9.resourceKey = **/cdk/opensearch-service-migration/**/migration-assistance-stack.ts + +# Allow any AWS caller to access the OpenSearch Target Domain used in integration tests +# and is vpc-isolated +sonar.issue.ignore.multicriteria.ts10.ruleKey = typescript:S6270 +sonar.issue.ignore.multicriteria.ts10.resourceKey = **/cdk/opensearch-service-migration/**/opensearch-domain-stack.ts + + +# Ignore TODO occurrences. We have other tools to track those +sonar.issue.ignore.multicriteria.j1.ruleKey = java:S1135 +sonar.issue.ignore.multicriteria.j1.resourceKey = **/*.java + +# Ignore System.*.println statements, we use them thoughtfully to alert the user on startup especially +# when logging may be misconfigured. +sonar.issue.ignore.multicriteria.j2.ruleKey = java:S106 +sonar.issue.ignore.multicriteria.j2.resourceKey = **/*.java + +# Ignore rule java:S2658 - This rule flags the use of asserts to check the parameters of a public method. +# This is used for checks that are included in the application but are too expensive to perform at runtime. +sonar.issue.ignore.multicriteria.j3.ruleKey = java:S4274 +sonar.issue.ignore.multicriteria.j3.resourceKey = **/*.java + +# Ignore class naming conventions for versioned RFS classes which contain underscores +sonar.issue.ignore.multicriteria.j4.ruleKey = java:S101 +sonar.issue.ignore.multicriteria.j4.resourceKey = **/bulkload/version_*/*.java + +# Ignore constructor parameter limits for Java classes +sonar.issue.ignore.multicriteria.j5.ruleKey = java:S107 +sonar.issue.ignore.multicriteria.j5.resourceKey = **/*.java + +# Ignore SonarQube when it tries to remove curly braces that make the code easier to read +sonar.issue.ignore.multicriteria.j6.ruleKey = java:S1602 +sonar.issue.ignore.multicriteria.j6.resourceKey = **/*.java + +# Ignore class naming conventions for versioned RFS classes which contain underscores +sonar.issue.ignore.multicriteria.j7.ruleKey = java:S101 +sonar.issue.ignore.multicriteria.j7.resourceKey = **/bulkload/transformers/*.java + +# Ignore switch statement warnings when the number of cases is low. +sonar.issue.ignore.multicriteria.j8.ruleKey = java:S1301 +sonar.issue.ignore.multicriteria.j8.resourceKey = **/*.java + +# Ignore Singleton design pattern warnings +sonar.issue.ignore.multicriteria.j9.ruleKey = java:S6548 +sonar.issue.ignore.multicriteria.j9.resourceKey = **/*.java + +# Ignore "Remove this method and declare a constant for this value." +sonar.issue.ignore.multicriteria.j10.ruleKey = java:S3400 +sonar.issue.ignore.multicriteria.j10.resourceKey = **/*.java + +# Ignore "This block of commented-out lines of code should be removed." +sonar.issue.ignore.multicriteria.j11.ruleKey = java:S125 +sonar.issue.ignore.multicriteria.j11.resourceKey = **/*.java + +# Ignore "Remove usage of generic wildcard type." - we had 8 places where we return values w/ wildcards. +# I don't see an easy workaround that keeps the same semantics. It's a shame that the wildcard detection +# isn't more sophisticated to tell when this is actually an issue. +sonar.issue.ignore.multicriteria.j12.ruleKey = java:S1452 +sonar.issue.ignore.multicriteria.j12.resourceKey = **/*.java + + + +# "Use try-with-resources or close this" +# We use AutoCloseable for tracing contexts so that we can properly record spans +# and metrics at the completion of activities. Since many of our contexts tend to +# be closed in code that runs asynchronously from when it was created, this makes +# a try-with-resources block a non-starter. It's unfortunate that there isn't +# another way to check all other auto-closeable values OTHER than instrumentation +# contexts. +# +# This is a good one to run audits on to make sure that we have close paths setup. +# It's also good to make sure that we're dealing with other AutoCloseables as effectively +# as possible. +sonar.issue.ignore.multicriteria.autoclose.ruleKey = java:S2095 +sonar.issue.ignore.multicriteria.autoclose.resourceKey = **/TrafficCapture/**/*.java + + + +# Cognitive complexity scores are too high for the files specified below. +# In most cases, the code is for specific state machines and very carefully constructed code. +# Refactoring to maintain correctness and efficiency would be nice, but making the solution require +# LESS cognitive load will be a sizeable challenge! +sonar.issue.ignore.multicriteria.comp1.ruleKey = java:S3776 +sonar.issue.ignore.multicriteria.comp1.resourceKey = **/BlockingTrafficSource.java + +sonar.issue.ignore.multicriteria.comp2.ruleKey = java:S3776 +sonar.issue.ignore.multicriteria.comp2.resourceKey = **/NettyJsonBodyAccumulateHandler.java + +sonar.issue.ignore.multicriteria.comp3.ruleKey = java:S3776 +sonar.issue.ignore.multicriteria.comp3.resourceKey = **/NettyJsonToByteBufHandler.java + +sonar.issue.ignore.multicriteria.comp4.ruleKey = java:S3776 +sonar.issue.ignore.multicriteria.comp4.resourceKey = **/HeaderRemoverHandler.java + + + +# Some loops are better with multiple exits +sonar.issue.ignore.multicriteria.loop1.ruleKey = java:S135 +sonar.issue.ignore.multicriteria.loop1.resourceKey = **/OpenSearchDefaultRetry.java + +sonar.issue.ignore.multicriteria.loop2.ruleKey = java:S135 +sonar.issue.ignore.multicriteria.loop2.resourceKey = **/TrafficReplayerCore.java + +sonar.issue.ignore.multicriteria.loop3.ruleKey = java:S135 +sonar.issue.ignore.multicriteria.loop3.resourceKey = **/TrafficReplayerTopLevel.java + +sonar.issue.ignore.multicriteria.loop4.ruleKey = java:S135 +sonar.issue.ignore.multicriteria.loop4.resourceKey = **/HeaderRemoverHandler.java + +sonar.issue.ignore.multicriteria.loop5.ruleKey = java:S135 +sonar.issue.ignore.multicriteria.loop5.resourceKey = **/NettyJsonToByteBufHandler.java + + + +# Ignore "Define a constant instead of duplicating this literal..." +sonar.issue.ignore.multicriteria.f1.ruleKey = java:S1192 +sonar.issue.ignore.multicriteria.f1.resourceKey = **/OpenSearchWorkCoordinator.java + +# Ignore empty class +sonar.issue.ignore.multicriteria.f2.ruleKey = java:S2094 +sonar.issue.ignore.multicriteria.f2.resourceKey = **/replay/**/EndOfInput.java + +# Throwable and Error should not be caught +# The Top-Level traffic replayer catches Throwables to orchestrates a shutdown +sonar.issue.ignore.multicriteria.f3.ruleKey = java:S1181 +sonar.issue.ignore.multicriteria.f3.resourceKey = **/TrafficReplayerTopLevel.java + +# Empty arrays and collections should be returned instead of null +# Testing for empty would require more code to be safe, so that's a bad idea +sonar.issue.ignore.multicriteria.f4.ruleKey = java:S1168 +sonar.issue.ignore.multicriteria.f4.resourceKey = **/TrafficStreamUtils.java + +# "/" is a URI path delimiter and it's more readable than any other option +sonar.issue.ignore.multicriteria.f5.ruleKey = java:S1075 +sonar.issue.ignore.multicriteria.f5.resourceKey = **/OpenSearchClient.java + +# A single extension of a netty class makes the class inheritance depth go deeper than Sonar would like +sonar.issue.ignore.multicriteria.f6.ruleKey = java:S110 +sonar.issue.ignore.multicriteria.f6.resourceKey = **/LoggingHttpHandler.java + +# Ignore "Remove this useless assignment to local variable...". +# The code uses literal variable names to help with readability +sonar.issue.ignore.multicriteria.f7.ruleKey = java:S1854 +sonar.issue.ignore.multicriteria.f7.resourceKey = **/ReplayEngine.java + +# Ignore 'Do something with the "!Unknown!" value returned by "tryAcquire"' +# There is a call to tryAcquire on a semaphore whose return value isn't tested +# because we immediately test for the condition again within a loop. +# There's no reason to add more branches to already complicated and sensitive code. +sonar.issue.ignore.multicriteria.f8.ruleKey = java:S899 +sonar.issue.ignore.multicriteria.f8.resourceKey = **/BlockingTrafficSource.java + + +# We use some base Docker images that run as root; changing the default user in these contaiers +# doesn't make sense +sonar.issue.ignore.multicriteria.d1.ruleKey = docker:S6471 +sonar.issue.ignore.multicriteria.d1.resourceKey = **/Dockerfile \ No newline at end of file