diff --git a/docs/getting-started/new-project-guide/go_lang.md b/docs/getting-started/new-project-guide/go_lang.md index f64e6913c7e4..59057ef7e532 100644 --- a/docs/getting-started/new-project-guide/go_lang.md +++ b/docs/getting-started/new-project-guide/go_lang.md @@ -27,10 +27,47 @@ only. In that mode, fuzz targets for Go use the libFuzzer engine with native Go coverage instrumentation. Binaries compiled in this mode provide the same libFuzzer command line interface as non-Go fuzz targets. +## Native Go Fuzzing support + +OSS-fuzz supports fuzzers written for the native Go 1.18 engine. These fuzzers are built as libFuzzer binaries in a similar fashion as fuzzers written for the go-fuzz engine. Because of that, dictionaries and seed corpora should be handled in accordance with [the OSS-fuzz documentation](https://google.github.io/oss-fuzz/getting-started/new-project-guide/#seed-corpus). +Unlike libFuzzer/go-fuzz targets which must accept one data buffer, fuzz targets written for the Native Go engine can accept any number of arguments of any type. Here is an example of a valid fuzzer with multiple arguments: + +```go +package demofuzzing + +import ( + "fmt" + "testing" +) + +func FuzzDemo(f *testing.F) { + f.Fuzz(func(t *testing.T, data1 string, data2 uint32, data3 float64) { + fmt.Println(data1) + fmt.Println(data2) + fmt.Println(data3) + }) +} +``` + +Some requirements for native Go 1.18 fuzzers are: +* The only `testing.F` method supported is currently `F.Fuzz()`. +* `F.Add()` will not add seeds when fuzzing. To provide OSS-fuzz with a seed corpus, follow the documentation [here](https://google.github.io/oss-fuzz/getting-started/new-project-guide/#seed-corpus). + +### Troubleshooting +```console +main.1320908145.go:8:2: found packages nativefuzzing fuzzer_test.go_fuzz_.go) and main (main.1320908145.go) in /src/project/go/test/fuzzing/nativefuzzing +``` + +This issue occurs because the cwd is a directory with a non-main package when running `compile_native_go_fuzzer`. To solve it, create a temporary directory in your project tree and `cd` into it before running `compile_native_go_fuzzer`: + +```sh +mkdir tmp && cd tmp +compile_native_go_fuzzer $path $fuzz_function $binary_name +``` + ## Project files -First, you need to write a Go fuzz target that accepts a stream of bytes and -calls the program API with that. This fuzz target should reside in your project +First, you need to write a Go fuzz target. This fuzz target should reside in your project repository ([example](https://github.com/golang/go/blob/4ad13555184eb0697c2e92c64c1b0bdb287ccc10/src/html/fuzz.go#L13)). @@ -79,8 +116,7 @@ In order to build a Go fuzz target, you need to call `go-fuzz` command first, and then link the resulting `.a` file against `$LIB_FUZZING_ENGINE` using the `$CXX $CXXFLAGS ...` command. -The best way to do this is by using a `compile_go_fuzzer` script, -as it also supports coverage builds. +For go-fuzz fuzzers, the best way to do this is by using the `compile_go_fuzzer` script, and for native Go 1.18 fuzzers it is recommended to use the `compile_native_go_fuzzer` script. Both of these also support coverage builds. A usage example from go-dns project is diff --git a/infra/base-images/base-builder-go/Dockerfile b/infra/base-images/base-builder-go/Dockerfile index 9d2c6150278a..5b4e01bb54ec 100644 --- a/infra/base-images/base-builder-go/Dockerfile +++ b/infra/base-images/base-builder-go/Dockerfile @@ -26,5 +26,6 @@ ENV PATH $PATH:/root/.go/bin:$GOPATH/bin RUN install_go.sh # TODO(jonathanmetzman): Install this file using install_go.sh. -COPY ossfuzz_coverage_runner.go $GOPATH - +COPY ossfuzz_coverage_runner.go \ + native_ossfuzz_coverage_runner.go \ + $GOPATH/ diff --git a/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go b/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go new file mode 100644 index 000000000000..1e26d8b23667 --- /dev/null +++ b/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go @@ -0,0 +1,71 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mypackagebeingfuzzed + +import ( + "io/ioutil" + "os" + "runtime/pprof" + "testing" + "github.com/AdamKorcz/go-118-fuzz-build/utils" +) + +func TestFuzzCorpus(t *testing.T) { + dir := os.Getenv("FUZZ_CORPUS_DIR") + if dir == "" { + t.Logf("No fuzzing corpus directory set") + return + } + infos, err := ioutil.ReadDir(dir) + if err != nil { + t.Logf("Not fuzzing corpus directory %s", err) + return + } + filename := "" + defer func() { + if r := recover(); r != nil { + t.Error("Fuzz panicked in "+filename, r) + } + }() + profname := os.Getenv("FUZZ_PROFILE_NAME") + if profname != "" { + f, err := os.Create(profname + ".cpu.prof") + if err != nil { + t.Logf("error creating profile file %s\n", err) + } else { + _ = pprof.StartCPUProfile(f) + } + } + for i := range infos { + filename = dir + infos[i].Name() + data, err := ioutil.ReadFile(filename) + if err != nil { + t.Error("Failed to read corpus file", err) + } + fuzzerF := &utils.F{Data:data, T:&testing.T{}} + FuzzFunction(fuzzerF) + } + if profname != "" { + pprof.StopCPUProfile() + f, err := os.Create(profname + ".heap.prof") + if err != nil { + t.Logf("error creating heap profile file %s\n", err) + } + if err = pprof.WriteHeapProfile(f); err != nil { + t.Logf("error writing heap profile file %s\n", err) + } + f.Close() + } +} diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 475e5a7d1134..02064fba97af 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -142,7 +142,9 @@ COPY precompile_honggfuzz /usr/local/bin/ RUN precompile_honggfuzz COPY cargo compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \ - compile_go_fuzzer debug_afl srcmap \ + compile_go_fuzzer \ + compile_native_go_fuzzer \ + debug_afl srcmap \ write_labels.py bazel_build_fuzz_tests \ # Go, java, and swift installation scripts. install_go.sh \ diff --git a/infra/base-images/base-builder/compile_native_go_fuzzer b/infra/base-images/base-builder/compile_native_go_fuzzer new file mode 100755 index 000000000000..5c29738d537c --- /dev/null +++ b/infra/base-images/base-builder/compile_native_go_fuzzer @@ -0,0 +1,97 @@ +#!/bin/bash -eu +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# Rewrites a copy of the fuzzer to allow for +# libFuzzer instrumentation. +function rewrite_go_fuzz_harness() { + fuzzer_filename=$1 + fuzz_function=$2 + + # Create a copy of the fuzzer to not modify the existing fuzzer. + cp $fuzzer_filename "${fuzzer_filename}"_fuzz_.go + mv $fuzzer_filename /tmp/ + fuzzer_fn="${fuzzer_filename}"_fuzz_.go + + # Replace *testing.F with *go118fuzzbuildutils.F. + echo "replacing *testing.F" + sed -i "s/func $fuzz_function(\([a-zA-Z0-9]*\) \*testing\.F)/func $fuzz_function(\1 \*go118fuzzbuildutils\.F)/g" "${fuzzer_fn}" + + # Import https://github.com/AdamKorcz/go-118-fuzz-build. + # This changes the line numbers from the original fuzzer. + addimport -path "${fuzzer_fn}" +} + +function build_native_go_fuzzer() { + fuzzer=$1 + function=$2 + path=$3 + tags="-tags gofuzz" + + if [[ $SANITIZER = *coverage* ]]; then + echo "here we perform coverage build" + fuzzed_package=`go list $tags -f '{{.Name}}' $path` + abspath=`go list $tags -f {{.Dir}} $path` + cd $abspath + cp $GOPATH/native_ossfuzz_coverage_runner.go ./"${function,,}"_test.go + sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go + sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go + sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go + + # The repo is the module path/name, which is already created above + # in case it doesn't exist, but not always the same as the module + # path. This is necessary to handle SIV properly. + fuzzed_repo=$(go list $tags -f {{.Module}} "$path") + abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo` + # give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir + echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath + gotip test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -c -o $OUT/$fuzzer $path + + rm ./"${function,,}"_test.go + else + go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir + $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer + fi +} + + +path=$1 +function=$2 +fuzzer=$3 +tags="-tags gofuzz" + +# Get absolute path. +abs_file_dir=$(go list $tags -f {{.Dir}} $path) + +# TODO(adamkorcz): Get rid of "-r" flag here. +fuzzer_filename=$(grep -r -l -s "$function" "${abs_file_dir}") + +# Test if file contains a line with "func $function" and "testing.F". +if [ $(grep -r "func $function" $fuzzer_filename | grep "testing.F" | wc -l) -eq 1 ] +then + # Install more dependencies. + gotip get github.com/AdamKorcz/go-118-fuzz-build/utils + gotip get google.golang.org/grpc/internal/channelz@v1.42.0 + + rewrite_go_fuzz_harness $fuzzer_filename $function + build_native_go_fuzzer $fuzzer $function $abs_file_dir + + # Clean up. + rm "${fuzzer_filename}_fuzz_.go" + mv /tmp/$(basename $fuzzer_filename) $fuzzer_filename +else + echo "Could not find the function: func ${function}(f *testing.F)" +fi diff --git a/infra/base-images/base-builder/install_go.sh b/infra/base-images/base-builder/install_go.sh index 21138831c1d7..0e8c9f04b8cc 100755 --- a/infra/base-images/base-builder/install_go.sh +++ b/infra/base-images/base-builder/install_go.sh @@ -26,3 +26,16 @@ echo 'Set "PATH=$PATH:/root/.go/bin:$GOPATH/bin"' go get -u github.com/mdempsky/go114-fuzz-build ln -s $GOPATH/bin/go114-fuzz-build $GOPATH/bin/go-fuzz + +go install golang.org/dl/gotip@latest \ + && gotip download + +cd /tmp +git clone https://github.com/AdamKorcz/go-118-fuzz-build +cd go-118-fuzz-build +gotip build +mv go-118-fuzz-build $GOPATH/bin/ + +cd addimport +gotip build +mv addimport $GOPATH/bin/ diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage index 8312f38ce64c..600bc625b652 100755 --- a/infra/base-images/base-runner/coverage +++ b/infra/base-images/base-runner/coverage @@ -129,7 +129,15 @@ function run_go_fuzz_target { echo "Running go target $target" export FUZZ_CORPUS_DIR="$CORPUS_DIR/${target}/" export FUZZ_PROFILE_NAME="$DUMPS_DIR/$target.perf" + $OUT/$target -test.coverprofile $DUMPS_DIR/$target.profdata &> $LOGS_DIR/$target.log + + # The Go 1.18 fuzzers are renamed to "*_fuzz_.go" during "infra/helper.py build_fuzzers". + # They are are therefore refered to as "*_fuzz_.go" in the profdata files. + # Since the copies named "*_fuzz_.go" do not exist in the file tree during + # the coverage build, we change the references in the .profdata files + # to the original file names. + sed -i "s/_test.go_fuzz_.go/_test.go/g" $DUMPS_DIR/$target.profdata # translate from golangish paths to current absolute paths cat $OUT/$target.gocovpath | while read i; do sed -i $i $DUMPS_DIR/$target.profdata; done # cf PATH_EQUIVALENCE_ARGS @@ -227,7 +235,9 @@ wait if [[ $FUZZING_LANGUAGE == "go" ]]; then $SYSGOPATH/bin/gocovmerge $DUMPS_DIR/*.profdata > fuzz.cov - go tool cover -html=fuzz.cov -o $REPORT_ROOT_DIR/index.html + # TODO(adamkorcz): Once oss-fuzz upgrades go to 1.18, + # use "go" instead of "gotip": + gotip tool cover -html=fuzz.cov -o $REPORT_ROOT_DIR/index.html $SYSGOPATH/bin/gocovsum fuzz.cov > $SUMMARY_FILE cp $REPORT_ROOT_DIR/index.html $REPORT_PLATFORM_DIR/index.html $SYSGOPATH/bin/pprof-merge $DUMPS_DIR/*.perf.cpu.prof