From cce0deb64b9c69b991263b5de5b90d9747c9fb96 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 2 Sep 2022 13:37:08 +0100 Subject: [PATCH 1/3] fuzz: Ensure latest base images are used Latest base image should contain Go 1.18, removing the need of updating that ourselves, apart from benefiting from latest changes upstream. Signed-off-by: Paulo Gomes --- Makefile | 2 +- tests/fuzz/Dockerfile.builder | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 9c0d51ed9..67360b873 100644 --- a/Makefile +++ b/Makefile @@ -151,7 +151,7 @@ fuzz-build: rm -rf $(BUILD_DIR)/fuzz/ mkdir -p $(BUILD_DIR)/fuzz/out/ - docker build . --tag local-fuzzing:latest -f tests/fuzz/Dockerfile.builder + docker build . --pull --tag local-fuzzing:latest -f tests/fuzz/Dockerfile.builder docker run --rm \ -e FUZZING_LANGUAGE=go -e SANITIZER=address \ -e CIFUZZ_DEBUG='True' -e OSS_FUZZ_PROJECT_NAME=fluxcd \ diff --git a/tests/fuzz/Dockerfile.builder b/tests/fuzz/Dockerfile.builder index 10296a5cd..43acfe5c5 100644 --- a/tests/fuzz/Dockerfile.builder +++ b/tests/fuzz/Dockerfile.builder @@ -1,10 +1,5 @@ -FROM golang:1.18 AS go - FROM gcr.io/oss-fuzz-base/base-builder-go -# ensures golang 1.18 to enable go native fuzzing. -COPY --from=go /usr/local/go /usr/local/ - COPY ./ $GOPATH/src/github.com/fluxcd/helm-controller/ COPY ./tests/fuzz/oss_fuzz_build.sh $SRC/build.sh From 5b371de9ead95db38e9d12056226e79a688dd248 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 2 Sep 2022 15:59:24 +0100 Subject: [PATCH 2/3] fuzz: Fix upstream build The upstream build is failing with: /usr/local/bin/compile_native_go_fuzzer: line 35: addimport: command not found The changes uses the upstream approach to install the missing commands. Note that one of the reasons of the unilateral failure upstream is due to the base image changed in: https://github.com/google/oss-fuzz/commit/e71e320709b7249348a2115a18481dd623b27a40 Signed-off-by: Paulo Gomes --- tests/fuzz/Dockerfile.builder | 5 -- tests/fuzz/compile_native_go_fuzzer.sh | 102 ------------------------- tests/fuzz/oss_fuzz_build.sh | 46 ++++++++++- 3 files changed, 42 insertions(+), 111 deletions(-) delete mode 100755 tests/fuzz/compile_native_go_fuzzer.sh diff --git a/tests/fuzz/Dockerfile.builder b/tests/fuzz/Dockerfile.builder index 43acfe5c5..0e8cbaf3e 100644 --- a/tests/fuzz/Dockerfile.builder +++ b/tests/fuzz/Dockerfile.builder @@ -3,9 +3,4 @@ FROM gcr.io/oss-fuzz-base/base-builder-go COPY ./ $GOPATH/src/github.com/fluxcd/helm-controller/ COPY ./tests/fuzz/oss_fuzz_build.sh $SRC/build.sh -# Temporarily overrides compile_native_go_fuzzer. -# Pending upstream merge: https://github.com/google/oss-fuzz/pull/8285 -COPY tests/fuzz/compile_native_go_fuzzer.sh /usr/local/bin/compile_native_go_fuzzer -RUN go install golang.org/x/tools/cmd/goimports@latest - WORKDIR $SRC diff --git a/tests/fuzz/compile_native_go_fuzzer.sh b/tests/fuzz/compile_native_go_fuzzer.sh deleted file mode 100755 index 8983d9dfd..000000000 --- a/tests/fuzz/compile_native_go_fuzzer.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/bash -eu -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -# Rewrites a copy of the fuzzer to allow for -# libFuzzer instrumentation. -function rewrite_go_fuzz_harness() { - fuzzer_filename=$1 - fuzz_function=$2 - - # Create a copy of the fuzzer to not modify the existing fuzzer. - cp $fuzzer_filename "${fuzzer_filename}"_fuzz_.go - mv $fuzzer_filename /tmp/ - fuzzer_fn="${fuzzer_filename}"_fuzz_.go - - # Remove the body of go testing funcs that may be co-located. - echo "removing *testing.T" - sed -i -e '/testing.T) {$/ {:r;/\n}/!{N;br}; s/\n.*\n/\n/}' "${fuzzer_fn}" - # After removing the body of the go testing funcs, consolidate the imports. - if command -v goimports; then - goimports -w "${fuzzer_fn}" - fi - - # Replace *testing.F with *go118fuzzbuildutils.F. - echo "replacing *testing.F" - sed -i "s/func $fuzz_function(\([a-zA-Z0-9]*\) \*testing\.F)/func $fuzz_function(\1 \*go118fuzzbuildutils\.F)/g" "${fuzzer_fn}" - - # Import https://github.com/AdamKorcz/go-118-fuzz-build. - # This changes the line numbers from the original fuzzer. - addimport -path "${fuzzer_fn}" -} - -function build_native_go_fuzzer() { - fuzzer=$1 - function=$2 - path=$3 - tags="-tags gofuzz" - - if [[ $SANITIZER = *coverage* ]]; then - echo "here we perform coverage build" - fuzzed_package=`go list $tags -f '{{.Name}}' $path` - abspath=`go list $tags -f {{.Dir}} $path` - cd $abspath - cp $GOPATH/native_ossfuzz_coverage_runner.go ./"${function,,}"_test.go - sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go - sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go - sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go - - # The repo is the module path/name, which is already created above - # in case it doesn't exist, but not always the same as the module - # path. This is necessary to handle SIV properly. - fuzzed_repo=$(go list $tags -f {{.Module}} "$path") - abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo` - # give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir - echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath - gotip test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -c -o $OUT/$fuzzer $path - - rm ./"${function,,}"_test.go - else - go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir - $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer - fi -} - - -path=$1 -function=$2 -fuzzer=$3 -tags="-tags gofuzz" - -# Get absolute path. -abs_file_dir=$(go list $tags -f {{.Dir}} $path) - -# TODO(adamkorcz): Get rid of "-r" flag here. -fuzzer_filename=$(grep -r -l --include='**.go' -s "$function" "${abs_file_dir}") - -# Test if file contains a line with "func $function" and "testing.F". -if [ $(grep -r "func $function" $fuzzer_filename | grep "testing.F" | wc -l) -eq 1 ] -then - - rewrite_go_fuzz_harness $fuzzer_filename $function - build_native_go_fuzzer $fuzzer $function $abs_file_dir - - # Clean up. - rm "${fuzzer_filename}_fuzz_.go" - mv /tmp/$(basename $fuzzer_filename) $fuzzer_filename -else - echo "Could not find the function: func ${function}(f *testing.F)" -fi diff --git a/tests/fuzz/oss_fuzz_build.sh b/tests/fuzz/oss_fuzz_build.sh index 9e4a36f2f..1b8351934 100755 --- a/tests/fuzz/oss_fuzz_build.sh +++ b/tests/fuzz/oss_fuzz_build.sh @@ -19,23 +19,61 @@ set -euxo pipefail GOPATH="${GOPATH:-/root/go}" GO_SRC="${GOPATH}/src" PROJECT_PATH="github.com/fluxcd/helm-controller" +TMP_DIR=$(mktemp -d /tmp/oss_fuzz-XXXXXX) + +cleanup(){ + rm -rf "${TMP_DIR}" +} +trap cleanup EXIT + +install_deps(){ + if ! command -v go-118-fuzz-build &> /dev/null || ! command -v addimport &> /dev/null; then + mkdir -p "${TMP_DIR}/go-118-fuzz-build" + + git clone https://github.com/AdamKorcz/go-118-fuzz-build "${TMP_DIR}/go-118-fuzz-build" + cd "${TMP_DIR}/go-118-fuzz-build" + go build -o "${GOPATH}/bin/go-118-fuzz-build" + + cd addimport + go build -o "${GOPATH}/bin/addimport" + fi + + if ! command -v goimports &> /dev/null; then + go install golang.org/x/tools/cmd/goimports@latest + fi +} + +# Removes the content of test funcs which could cause the Fuzz +# tests to break. +remove_test_funcs(){ + filename=$1 + + echo "removing co-located *testing.T" + sed -i -e '/func Test.*testing.T) {$/ {:r;/\n}/!{N;br}; s/\n.*\n/\n/}' "${filename}" + + # After removing the body of the go testing funcs, consolidate the imports. + goimports -w "${filename}" +} + +install_deps cd "${GO_SRC}/${PROJECT_PATH}" -go install github.com/AdamKorcz/go-118-fuzz-build@latest go get github.com/AdamKorcz/go-118-fuzz-build/utils # Iterate through all Go Fuzz targets, compiling each into a fuzzer. test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' .) for file in ${test_files} do + remove_test_funcs "${file}" + targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}") for target_name in ${targets} do - fuzzer_name=$(echo "${target_name}" | tr '[:upper:]' '[:lower:]') - target_dir=$(dirname "${file}") + fuzzer_name=$(echo "${target_name}" | tr '[:upper:]' '[:lower:]') + target_dir=$(dirname "${file}") echo "Building ${file}.${target_name} into ${fuzzer_name}" - compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}" fuzz + compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}" done done From c435ad994b93684312b2c1089b9bf2bf02d9df41 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 2 Sep 2022 16:00:20 +0100 Subject: [PATCH 3/3] fuzz: Reuse go cache from host Signed-off-by: Paulo Gomes --- .github/workflows/cifuzz.yaml | 8 ++++++-- Makefile | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cifuzz.yaml b/.github/workflows/cifuzz.yaml index e52caff2b..e0e35b043 100644 --- a/.github/workflows/cifuzz.yaml +++ b/.github/workflows/cifuzz.yaml @@ -13,12 +13,16 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: 1.18.x - name: Restore Go cache uses: actions/cache@v3 with: - path: /home/runner/work/_temp/_github_home/go/pkg/mod + path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-go- + ${{ runner.os }}-go - name: Smoke test Fuzzers run: make fuzz-smoketest diff --git a/Makefile b/Makefile index 67360b873..7ad51795c 100644 --- a/Makefile +++ b/Makefile @@ -155,6 +155,7 @@ fuzz-build: docker run --rm \ -e FUZZING_LANGUAGE=go -e SANITIZER=address \ -e CIFUZZ_DEBUG='True' -e OSS_FUZZ_PROJECT_NAME=fluxcd \ + -v "$(shell go env GOMODCACHE):/root/go/pkg/mod" \ -v "$(BUILD_DIR)/fuzz/out":/out \ local-fuzzing:latest