Skip to content

Commit

Permalink
Merge branch 'master' into amd/thread-count
Browse files Browse the repository at this point in the history
Features: dfuse
  • Loading branch information
ashleypittman committed Dec 15, 2023
2 parents 0d8cfff + 1014ae6 commit 2e784f4
Show file tree
Hide file tree
Showing 121 changed files with 5,962 additions and 5,357 deletions.
3 changes: 3 additions & 0 deletions .clang-format
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,6 @@ AlignConsecutiveBitFields: AcrossEmptyLinesAndComments
SpaceBeforeParens: ControlStatementsExceptForEachMacros
KeepEmptyLinesAtTheStartOfBlocks: false
MaxEmptyLinesToKeep: 1
---
Language: Json
IndentWidth: 2
30 changes: 30 additions & 0 deletions .github/workflows/bash_unit_testing.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
name: bash Unit Testing

on:
push:
pull_request:

concurrency:
group: bash-unit-test-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

defaults:
run:
shell: bash --noprofile --norc -ueo pipefail {0}

jobs:
Test-gha-functions:
name: Tests in ci/gha_functions.sh
runs-on: [self-hosted, light]
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Checkout bash_unit project
uses: actions/checkout@v4
with:
repository: 'pgrange/bash_unit'
path: bash_unit
- name: Unit testing with bash_unit
run: FORCE_COLOR=true ./bash_unit/bash_unit ci/gha_functions.sh
18 changes: 18 additions & 0 deletions .github/workflows/linting.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,24 @@ jobs:
- name: Run on build files.
run: find . -name SConscript | xargs isort --check-only

shell-check:
name: ShellCheck
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Run
run: sudo apt-get update && sudo apt-get install shellcheck
- name: Add error parser
run: echo -n "::add-matcher::ci/shellcheck-matcher.json"
- name: Run Shellcheck
# The check will run with this file from the target branch but the code from the PR so
# test for this file before calling it to prevent failures on PRs where this check is
# in the target branch but the PR is not updated to include it.
run: \[ ! -x ci/run_shellcheck.sh \] || ./ci/run_shellcheck.sh

log-check:
name: Logging macro checking
runs-on: ubuntu-22.04
Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/rpm-build-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ jobs:
echo "text<<EOF" >> $GITHUB_OUTPUT;
git show -s --format=%B | escape_single_quotes >> $GITHUB_OUTPUT;
echo "EOF" >> $GITHUB_OUTPUT;
- name: Identify Commit Pragmas
run: . ci/gha_functions.sh;
echo '${{steps.dequoted_commit_message.outputs.text }}' | get_commit_pragmas

Import-commit-pragmas:
name: Make commit pragma variables
Expand Down
60 changes: 0 additions & 60 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -400,66 +400,6 @@ pipeline {
expression { !skipStage() }
}
parallel {
stage('checkpatch') {
when {
beforeAgent true
expression { !skipStage() }
}
agent {
dockerfile {
filename 'Dockerfile.checkpatch'
dir 'utils/docker'
label 'docker_runner'
additionalBuildArgs dockerBuildArgs(add_repos: false)
}
}
steps {
checkPatch user: GITHUB_USER_USR,
password: GITHUB_USER_PSW,
ignored_files: 'src/control/vendor/*:' +
'*.pb-c.[ch]:' +
'src/client/java/daos-java/src/main/java/io/daos/dfs/uns/*:' +
'src/client/java/daos-java/src/main/java/io/daos/obj/attr/*:' +
/* groovylint-disable-next-line LineLength */
'src/client/java/daos-java/src/main/native/include/daos_jni_common.h:' +
'*.crt:' +
'*.pem:' +
'*_test.go:' +
'src/cart/_structures_from_macros_.h:' +
'src/tests/ftest/*.patch:' +
'src/tests/ftest/large_stdout.txt'
}
post {
always {
job_status_update()
/* when JENKINS-39203 is resolved, can probably use stepResult
here and remove the remaining post conditions
stepResult name: env.STAGE_NAME,
context: 'build/' + env.STAGE_NAME,
result: ${currentBuild.currentResult}
*/
}
/* temporarily moved some stuff into stepResult due to JENKINS-39203
failure {
githubNotify credentialsId: 'daos-jenkins-commit-status',
description: env.STAGE_NAME,
context: 'pre-build/' + env.STAGE_NAME,
status: 'ERROR'
}
success {
githubNotify credentialsId: 'daos-jenkins-commit-status',
description: env.STAGE_NAME,
context: 'pre-build/' + env.STAGE_NAME,
status: 'SUCCESS'
}
unstable {
githubNotify credentialsId: 'daos-jenkins-commit-status',
description: env.STAGE_NAME,
context: 'pre-build/' + env.STAGE_NAME,
status: 'FAILURE'
} */
}
} // stage('checkpatch')
stage('Python Bandit check') {
when {
beforeAgent true
Expand Down
2 changes: 1 addition & 1 deletion ci/coverity_unsuccessful.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ rm -f coverity/*
mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
ci_envs="$mydir/parse_ci_envs.sh"
if [ -e "${ci_envs}" ]; then
# shellcheck disable=SC1091
# shellcheck source=parse_ci_envs.sh
source "${ci_envs}"
fi

Expand Down
28 changes: 14 additions & 14 deletions ci/daos-doxygen-matcher.json
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
{
"problemMatcher": [
{
"owner": "daos-doxygen",
"pattern": [
{
"regexp": "(src/include/.+\\.h):(\\d+):\\s(warning|error):\\s(.+)$",
"file": 1,
"line": 2,
"severity": 3,
"message": 4
}
]
}
]
"problemMatcher": [
{
"owner": "daos-doxygen",
"pattern": [
{
"regexp": "(src/include/.+\\.h):(\\d+):\\s(warning|error):\\s(.+)$",
"file": 1,
"line": 2,
"severity": 3,
"message": 4
}
]
}
]
}
14 changes: 7 additions & 7 deletions ci/daos-flake-matcher.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
"owner": "flake8-error-f",
"severity": "error",
"pattern": [
{
"regexp": "^([^:]+):(\\d+):(\\d+):\\s+(F\\d+\\s+.+)$",
"file": 1,
"line": 2,
"column": 3,
"message": 4
}
{
"regexp": "^([^:]+):(\\d+):(\\d+):\\s+(F\\d+\\s+.+)$",
"file": 1,
"line": 2,
"column": 3,
"message": 4
}
]
}
]
Expand Down
14 changes: 7 additions & 7 deletions ci/daos-flakee-matcher.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
"owner": "flake8-error-e",
"severity": "error",
"pattern": [
{
"regexp": "^([^:]+):(\\d+):(\\d+):\\s+(E\\d+\\s+.+)$",
"file": 1,
"line": 2,
"column": 3,
"message": 4
}
{
"regexp": "^([^:]+):(\\d+):(\\d+):\\s+(E\\d+\\s+.+)$",
"file": 1,
"line": 2,
"column": 3,
"message": 4
}
]
}
]
Expand Down
14 changes: 7 additions & 7 deletions ci/daos-flakew-matcher.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
"owner": "flake8-error-w",
"severity": "warning",
"pattern": [
{
"regexp": "^([^:]+):(\\d+):(\\d+):\\s+(W\\d+\\s+.+)$",
"file": 1,
"line": 2,
"column": 3,
"message": 4
}
{
"regexp": "^([^:]+):(\\d+):(\\d+):\\s+(W\\d+\\s+.+)$",
"file": 1,
"line": 2,
"column": 3,
"message": 4
}
]
}
]
Expand Down
19 changes: 12 additions & 7 deletions ci/gha_functions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ get_test_tags() {


get_commit_pragmas() {
sed -ne '/^[^ ]*: */s/\([^:]*\): *\(.*\)/\1 \2/p' | while read -r a b; do
sed -Ene 's/^([-[:alnum:]]+): *([-\._ [:alnum:]]+)$/\1 \2/p' | while read -r a b; do
echo -n "${a//-/_}" | tr '[:lower:]' '[:upper:]'
# escape special characters in the value
echo "=$b" | sed -e 's/\([<> ]\)/\\\1/g'
Expand Down Expand Up @@ -240,7 +240,7 @@ Stage Name: $stage_name\" > /root/job_info


# This is run under the unit test framework at https://github.com/pgrange/bash_unit/
# I.e. ../bash_unit/bash_unit ci/gha_functions.sh ci/gha_functions.sh
# I.e. ../bash_unit/bash_unit ci/gha_functions.sh
test_test_tag_and_features() {
# Simple Test-tag: test
assert_equals "$(CP_TEST_TAG="always_passes always_fails" get_test_tags "-hw")" "always_passes,-hw always_fails,-hw"
Expand All @@ -257,7 +257,12 @@ test_test_tag_and_features() {
test_get_commit_pragmas() {
local msg='Escape spaces also
'"'"'Will-not-be-a-pragma: false'"'"' should not be considered a commit
pragma, but:
Should-not-be-a-pragma: bar will be because it was not quoted.
Skip-func-test-leap15: false
RPM-test-version: 2.5.100-13.10036.g65926e32
Skip-PR-comments: true
Test-tag: always_passes always_fails
EL8-VM9-label: all_vm9
Expand All @@ -270,21 +275,21 @@ Required-githooks: true
Signed-off-by: Brian J. Murrell <[email protected]>
'
assert_equals "$(echo "$msg" | get_commit_pragmas)" 'SKIP_FUNC_TEST_LEAP15=false
assert_equals "$(echo "$msg" | get_commit_pragmas)" 'SHOULD_NOT_BE_A_PRAGMA=bar\ will\ be\ because\ it\ was\ not\ quoted.
SKIP_FUNC_TEST_LEAP15=false
RPM_TEST_VERSION=2.5.100-13.10036.g65926e32
SKIP_PR_COMMENTS=true
TEST_TAG=always_passes\ always_fails
EL8_VM9_LABEL=all_vm9
EL9_VM9_LABEL=all_vm9
LEAP15_VM9_LABEL=all_vm9
HW_MEDIUM_LABEL=new_icx5
HW_LARGE_LABEL=new_icx9
REQUIRED_GITHOOKS=true
SIGNED_OFF_BY=Brian\ J.\ Murrell\ \<[email protected]\>'
REQUIRED_GITHOOKS=true'

}

test_jenkins_curl() {
JENKINS_URL="${JENKINS_URL:-https://build.hpdd.intel.com/}"
assert_equals "$(QUIET=true VERBOSE=false jenkins_curl -X POST "${JENKINS_URL}api/xml" 3>&1 >/dev/null | grep '^X-Content-Type-Options:')" "X-Content-Type-Options: nosniff
"
assert_equals "$(QUIET=true VERBOSE=false jenkins_curl -X POST "${JENKINS_URL}api/xml" 3>&1 >/dev/null | tr -d '\r' | grep '^X-Content-Type-Options:')" "X-Content-Type-Options: nosniff"
}
8 changes: 0 additions & 8 deletions ci/patch_src_in_place

This file was deleted.

2 changes: 1 addition & 1 deletion ci/rpm/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ fi
mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
ci_envs="$mydir/../parse_ci_envs.sh"
if [ -e "${ci_envs}" ]; then
# shellcheck disable=SC1091,SC1090
# shellcheck source=parse_ci_envs.sh
source "${ci_envs}"
fi

Expand Down
2 changes: 1 addition & 1 deletion ci/rpm/build_success.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ set -uex
mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
ci_envs="$mydir/../parse_ci_envs.sh"
if [ -e "${ci_envs}" ]; then
# shellcheck source=../parse_ci_envs.sh
# shellcheck source=parse_ci_envs.sh
source "${ci_envs}"
fi

Expand Down
3 changes: 1 addition & 2 deletions ci/rpm/build_unsuccessful.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ set -uex
mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
ci_envs="$mydir/../parse_ci_envs.sh"
if [ -e "${ci_envs}" ]; then
# at some point we want to use: shellcheck source=ci/parse_ci_envs.sh
# shellcheck disable=SC1091,SC1090
# shellcheck source=parse_ci_envs.sh
source "${ci_envs}"
fi

Expand Down
13 changes: 13 additions & 0 deletions ci/run_shellcheck.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/bin/sh

set -e

echo Running shellcheck

# Find and run shellcheck on all shell scripts. Previously the `file` command was used
# to identify shell commands but that checks for a #! which some of our shell code does not
# use so go purely on filename.
# This depends on shellcheck 0.7.2 or above so works in GitHub actions but not on el8.8
find . \( -path ./.git -o -path ./venv -o -path ./build -o -path ./src/control/vendor \
-o -path ./install -o -path ./src/rdb/raft -type d \) -prune -o -name "*.sh" -exec \
shellcheck --source-path ci --external-sources --format gcc \{\} \+
17 changes: 17 additions & 0 deletions ci/shellcheck-matcher.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{
"problemMatcher": [
{
"owner": "shellcheck-problem-matcher",
"severity": "error",
"pattern": [
{
"regexp": "^(.*?):(\\d+):(\\d*):?\\s+(?:fatal\\s+)?(warning|error|note):\\s+(.*)$",
"file": 1,
"line": 2,
"column": 3,
"message": 5
}
]
}
]
}
1 change: 1 addition & 0 deletions ci/unit/test_main_node.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ if grep /mnt/daos\ /proc/mounts; then
fi
sudo mkdir -p /mnt/daos

# shellcheck disable=SC1091
source build/.build_vars.sh

sudo mkdir -p "${SL_SRC_DIR}"
Expand Down
Loading

0 comments on commit 2e784f4

Please sign in to comment.