[draft] Integrate native go fuzzing (#7055)

docs/getting-started/new-project-guide/go_lang.md

@@ -27,10 +27,47 @@ only. In that mode, fuzz targets for Go use the libFuzzer engine with native Go
 coverage instrumentation. Binaries compiled in this mode provide the same
 libFuzzer command line interface as non-Go fuzz targets.
 
+## Native Go Fuzzing support
+
+OSS-fuzz supports fuzzers written for the native Go 1.18 engine. These fuzzers are built as libFuzzer binaries in a similar fashion as fuzzers written for the go-fuzz engine. Because of that, dictionaries and seed corpora should be handled in accordance with [the OSS-fuzz documentation](https://google.github.io/oss-fuzz/getting-started/new-project-guide/#seed-corpus).
+Unlike libFuzzer/go-fuzz targets which must accept one data buffer, fuzz targets written for the Native Go engine can accept any number of arguments of any type. Here is an example of a valid fuzzer with multiple arguments:
+
+```go
+package demofuzzing
+
+import (
+    "fmt"
+    "testing"
+)
+
+func FuzzDemo(f *testing.F) {
+    f.Fuzz(func(t *testing.T, data1 string, data2 uint32, data3 float64) {
+        fmt.Println(data1)
+        fmt.Println(data2)
+        fmt.Println(data3)
+    })
+}
+```
+
+Some requirements for native Go 1.18 fuzzers are:
+* The only `testing.F` method supported is currently `F.Fuzz()`.
+* `F.Add()` will not add seeds when fuzzing. To provide OSS-fuzz with a seed corpus, follow the documentation [here](https://google.github.io/oss-fuzz/getting-started/new-project-guide/#seed-corpus).
+
+### Troubleshooting
+```console
+main.1320908145.go:8:2: found packages nativefuzzing fuzzer_test.go_fuzz_.go) and main (main.1320908145.go) in /src/project/go/test/fuzzing/nativefuzzing
+```
+
+This issue occurs because the cwd is a directory with a non-main package when running `compile_native_go_fuzzer`. To solve it, create a temporary directory in your project tree and `cd` into it before running `compile_native_go_fuzzer`:
+
+```sh
+mkdir tmp && cd tmp
+compile_native_go_fuzzer $path $fuzz_function $binary_name
+```
+
 ## Project files
 
-First, you need to write a Go fuzz target that accepts a stream of bytes and
-calls the program API with that. This fuzz target should reside in your project
+First, you need to write a Go fuzz target. This fuzz target should reside in your project
 repository
 ([example](https://github.com/golang/go/blob/4ad13555184eb0697c2e92c64c1b0bdb287ccc10/src/html/fuzz.go#L13)).
 
@@ -79,8 +116,7 @@ In order to build a Go fuzz target, you need to call `go-fuzz`
 command first, and then link the resulting `.a` file against
 `$LIB_FUZZING_ENGINE` using the `$CXX $CXXFLAGS ...` command.
 
-The best way to do this is by using a `compile_go_fuzzer` script,
-as it also supports coverage builds.
+For go-fuzz fuzzers, the best way to do this is by using the `compile_go_fuzzer` script, and for native Go 1.18 fuzzers it is recommended to use the `compile_native_go_fuzzer` script. Both of these also support coverage builds.
 
 A usage example from go-dns project is
 

infra/base-images/base-builder-go/Dockerfile

@@ -26,5 +26,6 @@ ENV PATH $PATH:/root/.go/bin:$GOPATH/bin
 RUN install_go.sh
 
 # TODO(jonathanmetzman): Install this file using install_go.sh.
-COPY ossfuzz_coverage_runner.go $GOPATH
-
+COPY ossfuzz_coverage_runner.go \
+     native_ossfuzz_coverage_runner.go \
+     $GOPATH/

infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go

@@ -0,0 +1,71 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mypackagebeingfuzzed
+
+import (
+	"io/ioutil"
+	"os"
+	"runtime/pprof"
+	"testing"
+	"github.com/AdamKorcz/go-118-fuzz-build/utils"
+)
+
+func TestFuzzCorpus(t *testing.T) {
+	dir := os.Getenv("FUZZ_CORPUS_DIR")
+	if dir == "" {
+		t.Logf("No fuzzing corpus directory set")
+		return
+	}
+	infos, err := ioutil.ReadDir(dir)
+	if err != nil {
+		t.Logf("Not fuzzing corpus directory %s", err)
+		return
+	}
+	filename := ""
+	defer func() {
+		if r := recover(); r != nil {
+			t.Error("Fuzz panicked in "+filename, r)
+		}
+	}()
+	profname := os.Getenv("FUZZ_PROFILE_NAME")
+	if profname != "" {
+		f, err := os.Create(profname + ".cpu.prof")
+		if err != nil {
+			t.Logf("error creating profile file %s\n", err)
+		} else {
+			_ = pprof.StartCPUProfile(f)
+		}
+	}
+	for i := range infos {
+		filename = dir + infos[i].Name()
+		data, err := ioutil.ReadFile(filename)
+		if err != nil {
+			t.Error("Failed to read corpus file", err)
+		}
+		fuzzerF := &utils.F{Data:data, T:&testing.T{}}
+		FuzzFunction(fuzzerF)
+	}
+	if profname != "" {
+		pprof.StopCPUProfile()
+		f, err := os.Create(profname + ".heap.prof")
+		if err != nil {
+			t.Logf("error creating heap profile file %s\n", err)
+		}
+		if err = pprof.WriteHeapProfile(f); err != nil {
+			t.Logf("error writing heap profile file %s\n", err)
+		}
+		f.Close()
+	}
+}

infra/base-images/base-builder/Dockerfile

@@ -146,7 +146,9 @@ COPY precompile_honggfuzz /usr/local/bin/
 RUN precompile_honggfuzz
 
 COPY cargo compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \
-    compile_go_fuzzer debug_afl srcmap \
+    compile_go_fuzzer \
+    compile_native_go_fuzzer \
+    debug_afl srcmap \
     write_labels.py bazel_build_fuzz_tests \
     # Go, java, and swift installation scripts.
     install_go.sh \

infra/base-images/base-builder/compile_native_go_fuzzer

@@ -0,0 +1,97 @@
+#!/bin/bash -eu
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# Rewrites a copy of the fuzzer to allow for 
+# libFuzzer instrumentation.
+function rewrite_go_fuzz_harness() {
+	fuzzer_filename=$1
+	fuzz_function=$2
+
+	# Create a copy of the fuzzer to not modify the existing fuzzer.
+	cp $fuzzer_filename "${fuzzer_filename}"_fuzz_.go
+	mv $fuzzer_filename /tmp/
+	fuzzer_fn="${fuzzer_filename}"_fuzz_.go
+
+	# Replace *testing.F with *go118fuzzbuildutils.F.
+	echo "replacing *testing.F"
+	sed -i "s/func $fuzz_function(\([a-zA-Z0-9]*\) \*testing\.F)/func $fuzz_function(\1 \*go118fuzzbuildutils\.F)/g" "${fuzzer_fn}"
+
+	# Import https://github.com/AdamKorcz/go-118-fuzz-build.
+	# This changes the line numbers from the original fuzzer.
+	addimport -path "${fuzzer_fn}"
+}
+
+function build_native_go_fuzzer() {
+	fuzzer=$1
+	function=$2
+	path=$3
+	tags="-tags gofuzz"
+
+	if [[ $SANITIZER = *coverage* ]]; then
+		echo "here we perform coverage build"
+		fuzzed_package=`go list $tags -f '{{.Name}}' $path`
+		abspath=`go list $tags -f {{.Dir}} $path`
+		cd $abspath
+		cp $GOPATH/native_ossfuzz_coverage_runner.go ./"${function,,}"_test.go
+		sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go
+		sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go
+		sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go
+
+		# The repo is the module path/name, which is already created above
+		# in case it doesn't exist, but not always the same as the module
+		# path. This is necessary to handle SIV properly.
+		fuzzed_repo=$(go list $tags -f {{.Module}} "$path")
+		abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo`
+		# give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir
+		echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath
+		gotip test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -c -o $OUT/$fuzzer $path
+
+		rm ./"${function,,}"_test.go
+	else
+		go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir
+		$CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer
+	fi
+}
+
+
+path=$1
+function=$2
+fuzzer=$3
+tags="-tags gofuzz"
+
+# Get absolute path.
+abs_file_dir=$(go list $tags -f {{.Dir}} $path)
+
+# TODO(adamkorcz): Get rid of "-r" flag here.
+fuzzer_filename=$(grep -r -l  -s "$function" "${abs_file_dir}")
+
+# Test if file contains a line with "func $function" and "testing.F".
+if [ $(grep -r "func $function" $fuzzer_filename | grep "testing.F" | wc -l) -eq 1 ]
+then
+	# Install more dependencies.
+	gotip get github.com/AdamKorcz/go-118-fuzz-build/utils
+	gotip get google.golang.org/grpc/internal/[email protected]
+
+	rewrite_go_fuzz_harness $fuzzer_filename $function
+	build_native_go_fuzzer $fuzzer $function $abs_file_dir
+
+	# Clean up.
+	rm "${fuzzer_filename}_fuzz_.go"
+	mv /tmp/$(basename $fuzzer_filename) $fuzzer_filename
+else
+	echo "Could not find the function: func ${function}(f *testing.F)"
+fi

infra/base-images/base-builder/install_go.sh

@@ -26,3 +26,16 @@ echo 'Set "PATH=$PATH:/root/.go/bin:$GOPATH/bin"'
 
 go get -u github.com/mdempsky/go114-fuzz-build
 ln -s $GOPATH/bin/go114-fuzz-build $GOPATH/bin/go-fuzz
+
+go install golang.org/dl/gotip@latest \
+    && gotip download
+
+cd /tmp
+git clone https://github.com/AdamKorcz/go-118-fuzz-build
+cd go-118-fuzz-build
+gotip build
+mv go-118-fuzz-build $GOPATH/bin/
+
+cd addimport
+gotip build
+mv addimport $GOPATH/bin/

infra/base-images/base-runner/coverage

@@ -133,7 +133,15 @@ function run_go_fuzz_target {
   echo "Running go target $target"
   export FUZZ_CORPUS_DIR="$CORPUS_DIR/${target}/"
   export FUZZ_PROFILE_NAME="$DUMPS_DIR/$target.perf"
+
   $OUT/$target -test.coverprofile $DUMPS_DIR/$target.profdata &> $LOGS_DIR/$target.log
+
+  # The Go 1.18 fuzzers are renamed to "*_fuzz_.go" during "infra/helper.py build_fuzzers".
+  # They are are therefore refered to as "*_fuzz_.go" in the profdata files.
+  # Since the copies named "*_fuzz_.go" do not exist in the file tree during
+  # the coverage build, we change the references in the .profdata files
+  # to the original file names.
+  sed -i "s/_test.go_fuzz_.go/_test.go/g" $DUMPS_DIR/$target.profdata
   # translate from golangish paths to current absolute paths
   cat $OUT/$target.gocovpath | while read i; do sed -i $i $DUMPS_DIR/$target.profdata; done
   # cf PATH_EQUIVALENCE_ARGS
@@ -231,7 +239,9 @@ wait
 
 if [[ $FUZZING_LANGUAGE == "go" ]]; then
   $SYSGOPATH/bin/gocovmerge $DUMPS_DIR/*.profdata > fuzz.cov
-  go tool cover -html=fuzz.cov -o $REPORT_ROOT_DIR/index.html
+  # TODO(adamkorcz): Once oss-fuzz upgrades go to 1.18, 
+  # use "go" instead of "gotip":
+  gotip tool cover -html=fuzz.cov -o $REPORT_ROOT_DIR/index.html
   $SYSGOPATH/bin/gocovsum fuzz.cov > $SUMMARY_FILE
   cp $REPORT_ROOT_DIR/index.html $REPORT_PLATFORM_DIR/index.html
   $SYSGOPATH/bin/pprof-merge $DUMPS_DIR/*.perf.cpu.prof