ci: ensure all nightlies post github issues when tests fail

build/teamcity/cockroach/nightlies/compose.sh

@@ -4,22 +4,35 @@ set -xeuo pipefail
 
 dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
 source "$dir/teamcity-support.sh"
+source "$dir/teamcity-bazel-support.sh"
 
 tc_start_block "Run compose tests"
 
-bazel build //pkg/cmd/bazci --config=ci
-BAZCI=$(bazel info bazel-bin --config=ci)/pkg/cmd/bazci/bazci_/bazci
+bazel build //pkg/cmd/bazci //pkg/cmd/github-post //pkg/cmd/testfilter --config=ci
+BAZEL_BIN=$(bazel info bazel-bin --config=ci)
+BAZCI=$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci
 
 bazel build //pkg/cmd/cockroach //pkg/compose/compare/compare:compare_test --config=ci --config=crosslinux --config=test --config=with_ui
 CROSSBIN=$(bazel info bazel-bin --config=ci --config=crosslinux --config=test --config=with_ui)
 COCKROACH=$CROSSBIN/pkg/cmd/cockroach/cockroach_/cockroach
 COMPAREBIN=$(bazel run //pkg/compose/compare/compare:compare_test --config=ci --config=crosslinux --config=test --config=with_ui --run_under=realpath | grep '^/' | tail -n1)
+ARTIFACTS_DIR=$PWD/artifacts
 
-$BAZCI run --config=ci --config=test --artifacts_dir=$PWD/artifacts \
+GO_TEST_JSON_OUTPUT_FILE=$PWD/artifacts/test.json.txt
+exit_status=0
+$BAZCI run --config=ci --config=test --artifacts_dir=$ARTIFACTS_DIR \
        //pkg/compose:compose_test -- \
        --test_env=GO_TEST_WRAP_TESTV=1 \
+       --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE \
        --test_arg -cockroach --test_arg $COCKROACH \
        --test_arg -compare --test_arg $COMPAREBIN \
-       --test_timeout=1800
+       --test_timeout=1800 || exit_status=$?
+process_test_json \
+        $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+        $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+        $ARTIFACTS_DIR \
+        $GO_TEST_JSON_OUTPUT_FILE \
+        $exit_status
 
 tc_end_block "Run compose tests"
+exit $exit_status

build/teamcity/cockroach/nightlies/lint_urls_impl.sh

@@ -2,10 +2,23 @@
 
 set -xeuo pipefail
 
-bazel build //pkg/cmd/bazci --config=ci
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+source "$dir/teamcity-bazel-support.sh"  # For process_test_json
+
+bazel build //pkg/cmd/bazci //pkg/cmd/github-post //pkg/cmd/testfilter --config=ci
+BAZEL_BIN=$(bazel info bazel-bin --config=ci)
+GO_TEST_JSON_OUTPUT_FILE=/artifacts/test.json.txt
+exit_status=0
 XML_OUTPUT_FILE=/artifacts/test.xml GO_TEST_WRAP_TESTV=1 GO_TEST_WRAP=1 bazel \
 	       run --config=ci --config=test --define gotags=bazel,gss,nightly \
-	       //build/bazelutil:lint
+	       //build/bazelutil:lint || exit_status=$?
 # The schema of the output test.xml will be slightly wrong -- ask `bazci` to fix
 # it up.
-$(bazel info bazel-bin --config=ci)/pkg/cmd/bazci/bazci_/bazci munge-test-xml /artifacts/test.xml
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci munge-test-xml /artifacts/test.xml
+process_test_json \
+    $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+    $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+    /artifacts \
+    $GO_TEST_JSON_OUTPUT_FILE \
+    $exit_status
+exit $exit_status

build/teamcity/cockroach/nightlies/optimizer_tests_impl.sh

@@ -3,26 +3,55 @@
 set -xeuo pipefail
 
 dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+source "$dir/teamcity-bazel-support.sh"
 source "$dir/teamcity/util.sh"
 
-bazel build //pkg/cmd/bazci --config=ci
+bazel build //pkg/cmd/bazci //pkg/cmd/github-post //pkg/cmd/testfilter --config=ci
 BAZEL_BIN=$(bazel info bazel-bin --config=ci)
 
 tc_start_block "Run opt tests with fast_int_set_large"
-$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \
+ARTIFACTS_DIR=/artifacts/fast_int_set_large
+mkdir $ARTIFACTS_DIR
+GO_TEST_JSON_OUTPUT_FILE=$ARTIFACTS_DIR/test.json.txt
+exit_status_large=0
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci --artifacts $ARTIFACTS_DIR \
     test //pkg/sql/opt:opt_test -- \
-    --define gotags=bazel,crdb_test,fast_int_set_large
-mkdir /artifacts/fast_int_set_large
-for FILE in $(ls artifacts | grep -v '^fast_int_set_large$'); do mv /artifacts/$FILE /artifacts/fast_int_set_large; done
+    --define gotags=bazel,crdb_test,fast_int_set_large \
+    --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE || $exit_status_large=$?
+process_test_json \
+        $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+        $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+        $ARTIFACTS_DIR \
+        $GO_TEST_JSON_OUTPUT_FILE \
+        $exit_status_large
 tc_end_block "Run opt tests with fast_int_set_large"
 
 # NOTE(ricky): Running both tests in the same configuration with different
 # gotags thrashes the cache. These tests are pretty quick so it shouldn't
 # matter now but it is something to keep an eye on.
 tc_start_block "Run opt tests with fast_int_set_small"
+ARTIFACTS_DIR=/artifacts/fast_int_set_small
+mkdir $ARTIFACTS_DIR
+GO_TEST_JSON_OUTPUT_FILE=$ARTIFACTS_DIR/test.json.txt
+exit_status_small=0
 $BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \
     test //pkg/sql/opt:opt_test -- \
-    --define gotags=bazel,crdb_test,fast_int_set_small
-mkdir /artifacts/fast_int_set_small
-for FILE in $(ls artifacts | grep -v '^fast_int_set'); do mv /artifacts/$FILE /artifacts/fast_int_set_small; done
+    --define gotags=bazel,crdb_test,fast_int_set_small \
+    --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE || $exit_status_small=$?
+process_test_json \
+        $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+        $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+        $ARTIFACTS_DIR \
+        $GO_TEST_JSON_OUTPUT_FILE \
+        $exit_status_large
 tc_end_block "Run opt tests with fast_int_set_small"
+
+if [ $exit_status_large -ne 0 ]
+then
+    exit $exit_status_large
+fi
+
+if [ $exit_status_small -ne 0 ]
+then
+    exit $exit_status_small
+fi

build/teamcity/cockroach/nightlies/random_syntax_tests_impl.sh

@@ -2,9 +2,22 @@
 
 set -xeuo pipefail
 
-bazel build //pkg/cmd/bazci --config=ci
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+source "$dir/teamcity-bazel-support.sh"
+
+bazel build //pkg/cmd/bazci //pkg/cmd/github-post //pkg/cmd/testfilter --config=ci
 BAZEL_BIN=$(bazel info bazel-bin --config=ci)
+GO_TEST_JSON_OUTPUT_FILE=/artifacts/test.json.txt
+exit_status=0
 $BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \
     test //pkg/sql/tests:tests_test -- \
     --test_arg -rsg=5m --test_arg -rsg-routines=8 --test_arg -rsg-exec-timeout=1m \
-    --test_timeout 3600 --test_filter 'TestRandomSyntax'
+    --test_timeout 3600 --test_filter 'TestRandomSyntax' \
+    --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE || exit_status=$?
+process_test_json \
+        $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+        $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+        /artifacts \
+        $GO_TEST_JSON_OUTPUT_FILE \
+        $exit_status
+exit $exit_status

build/teamcity/cockroach/nightlies/sqlite_logic_test_impl.sh

@@ -2,10 +2,23 @@
 
 set -xeuo pipefail
 
-bazel build //pkg/cmd/bazci --config=ci
+dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))"
+source "$dir/teamcity-bazel-support.sh"
+
+bazel build //pkg/cmd/bazci //pkg/cmd/github-post //pkg/cmd/testfilter --config=ci
 BAZEL_BIN=$(bazel info bazel-bin --config=ci)
+GO_TEST_JSON_OUTPUT_FILE=/artifacts/test.json.txt
+exit_status=0
 $BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci --config=ci \
     test //pkg/sql/logictest:logictest_test -- \
     --test_arg -bigtest --test_arg -flex-types \
     --define gotags=bazel,crdb_test_off --test_timeout 86400 \
-    --test_filter '^TestSqlLiteLogic$|^TestTenantSQLLiteLogic$'
+    --test_filter '^TestSqlLiteLogic$|^TestTenantSQLLiteLogic$' \
+    --test_env=GO_TEST_JSON_OUTPUT_FILE=$GO_TEST_JSON_OUTPUT_FILE || $exit_status=$?
+process_test_json \
+        $BAZEL_BIN/pkg/cmd/testfilter/testfilter_/testfilter \
+        $BAZEL_BIN/pkg/cmd/github-post/github-post_/github-post \
+        /artifacts \
+        $GO_TEST_JSON_OUTPUT_FILE \
+        $exit_status
+exit $exit_status