diff --git a/projects/vitess/Dockerfile b/projects/vitess/Dockerfile index 497af42f68db..19bbb8ab2b6a 100644 --- a/projects/vitess/Dockerfile +++ b/projects/vitess/Dockerfile @@ -19,5 +19,5 @@ RUN git clone --depth 1 https://github.com/vitessio/vitess RUN go install golang.org/dl/gotip@latest \ && gotip download RUN go install github.com/AdamKorcz/go-118-fuzz-build@latest -COPY build.sh tablet_manager_fuzzer_test.go parser_fuzzer_test.go ast_fuzzer_test.go $SRC/ +COPY build.sh native_ossfuzz_coverage_runnger.go tablet_manager_fuzzer_test.go parser_fuzzer_test.go ast_fuzzer_test.go $SRC/ WORKDIR $SRC/vitess diff --git a/projects/vitess/build.sh b/projects/vitess/build.sh index bfa338d1cb53..43d9ba68b849 100755 --- a/projects/vitess/build.sh +++ b/projects/vitess/build.sh @@ -29,6 +29,9 @@ cd $SRC/vitess # Remove existing non-native fuzzers to not deal with them rm go/vt/vtgate/vindexes/fuzz.go + +# backup vtctl_fuzzer.go +cp go/test/fuzzing/vtctl_fuzzer.go /tmp/ rm -r go/test/fuzzing/* mv $SRC/parser_fuzzer_test.go $SRC/vitess/go/test/fuzzing/ @@ -76,6 +79,7 @@ function rewrite_go_fuzz_harness() { # Create a copy of the fuzzer to not modify the existing fuzzer cp $fuzzer_filename "${fuzzer_filename}"_fuzz_.go + mv $fuzzer_filename /tmp/ # replace *testing.F with *go118fuzzbuildutils.F echo "replacing *testing.F" @@ -84,15 +88,6 @@ function rewrite_go_fuzz_harness() { # import https://github.com/AdamKorcz/go-118-fuzz-build # This changes the line numbers from the original fuzzer $SRC/go-118-fuzz-build/addimport/addimport -path "${fuzzer_filename}"_fuzz_.go - - # Install more dependencies - gotip get github.com/AdamKorcz/go-118-fuzz-build/utils - gotip get google.golang.org/grpc/internal/channelz@v1.39.0 -} - -function cleanup(){ - filename=$1 - rm $filename } function compile_native_go_fuzzer() { @@ -103,15 +98,34 @@ function compile_native_go_fuzzer() { if [[ $SANITIZER = *coverage* ]]; then echo "here we perform coverage build" + fuzzed_package=`go list $tags -f '{{.Name}}' $path` + abspath=`go list $tags -f {{.Dir}} $path` + cd $abspath + cp $SRC/native_ossfuzz_coverage_runnger.go ./"${function,,}"_test.go + sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go + sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go + sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go + + # The repo is the module path/name, which is already created above in case it doesn't exist, + # but not always the same as the module path. This is necessary to handle SIV properly. + fuzzed_repo=$(go list $tags -f {{.Module}} "$path") + abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo` + # give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir + echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath + ls + gotip test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -c -o $OUT/$fuzzer $path + + rm ./"${function,,}"_test.go else $SRC/go-118-fuzz-build/go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer fi } + # build_go_fuzzer will be the api used by users -# similar to compile_go_fuzzer. The api is now placed in -# this build script but will be moved to the base image -# once it has reached sufficient maturity. +# The api is now placed in this build script +# but will be moved to the base image once it +# has reached sufficient maturity. function build_go_fuzzer () { path=$1 function=$2 @@ -120,17 +134,6 @@ function build_go_fuzzer () { # Get absolute path abs_file_dir=$(go list $tags -f {{.Dir}} $path) - - # Check if there are more than 1 function named $function - number_of_occurences=$(grep -s "func $function" $abs_file_dir/{*,.*} | wc -l) - echo $number_of_occurences - if [ $number_of_occurences -ne 1 ] - then - echo "Found multiple targets '$function' in $path" - exit 0 - else - echo "GREAT! Only a single target was found" - fi # TODO: Get rid of "-r" flag here fuzzer_filename=$(grep -r -l -s "$function" "${abs_file_dir}") @@ -139,19 +142,26 @@ function build_go_fuzzer () { if [ $(grep -r "func $function" $fuzzer_filename | grep "testing.F" | wc -l) -eq 1 ] then # we are dealing with a native harness - echo "This is a native harness" + + # Install more dependencies + gotip get github.com/AdamKorcz/go-118-fuzz-build/utils + gotip get google.golang.org/grpc/internal/channelz@v1.39.0 + + echo "Native harness" rewrite_go_fuzz_harness $fuzzer_filename compile_native_go_fuzzer $fuzzer $function $abs_file_dir # clean up - cleanup "${fuzzer_filename}_fuzz_.go" + rm "${fuzzer_filename}_fuzz_.go" + mv /tmp/$(basename $fuzzer_filename) $fuzzer_filename else # we are dealing with a go-fuzz harness - echo "This is a go-fuzz harness" - # here we call compile_go_fuzzer + echo "go-fuzz harness" + compile_go_fuzzer $path $function $fuzzer $tags fi } +# build native fuzzers build_go_fuzzer vitess.io/vitess/go/test/fuzzing FuzzTabletManager_ExecuteFetchAsDba fuzz_tablet_manager_execute_fetch_as_dba build_go_fuzzer vitess.io/vitess/go/test/fuzzing FuzzParser parser_fuzzer build_go_fuzzer vitess.io/vitess/go/test/fuzzing FuzzIsDML is_dml_fuzzer @@ -159,3 +169,14 @@ build_go_fuzzer vitess.io/vitess/go/test/fuzzing FuzzNormalizer normalizer_fuzze build_go_fuzzer vitess.io/vitess/go/test/fuzzing FuzzNodeFormat normalizer_fuzzer build_go_fuzzer vitess.io/vitess/go/test/fuzzing FuzzSplitStatementToPieces fuzz_split_statement_to_pieces build_go_fuzzer vitess.io/vitess/go/test/fuzzing FuzzEqualsSQLNode fuzz_equals_sql_node + +# Delete all the native fuzzers before building the go-fuzz fuzzer(s) +# this will not be necessary when Go 1.18 is released. The reason this +# is needed is because go114-fuzz-build calls "go" instead of "gotip", +# and an error will be thrown because testing.F is not recognized. +rm $SRC/vitess/go/test/fuzzing/*_test.go + +# build go-fuzz fuzzers +mv /tmp/vtctl_fuzzer.go $SRC/vitess/go/test/fuzzing/ +build_go_fuzzer vitess.io/vitess/go/test/fuzzing Fuzz vtctl_fuzzer + diff --git a/projects/vitess/native_ossfuzz_coverage_runnger.go b/projects/vitess/native_ossfuzz_coverage_runnger.go new file mode 100644 index 000000000000..b7399e1918e6 --- /dev/null +++ b/projects/vitess/native_ossfuzz_coverage_runnger.go @@ -0,0 +1,71 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mypackagebeingfuzzed + +import ( + "io/ioutil" + "os" + "runtime/pprof" + "testing" + "github.com/AdamKorcz/go-118-fuzz-build/utils" +) + +func TestFuzzCorpus(t *testing.T) { + dir := os.Getenv("FUZZ_CORPUS_DIR") + if dir == "" { + t.Logf("No fuzzing corpus directory set") + return + } + infos, err := ioutil.ReadDir(dir) + if err != nil { + t.Logf("Not fuzzing corpus directory %s", err) + return + } + filename := "" + defer func() { + if r := recover(); r != nil { + t.Error("Fuzz panicked in "+filename, r) + } + }() + profname := os.Getenv("FUZZ_PROFILE_NAME") + if profname != "" { + f, err := os.Create(profname + ".cpu.prof") + if err != nil { + t.Logf("error creating profile file %s\n", err) + } else { + _ = pprof.StartCPUProfile(f) + } + } + for i := range infos { + filename = dir + infos[i].Name() + data, err := ioutil.ReadFile(filename) + if err != nil { + t.Error("Failed to read corpus file", err) + } + fuzzerF := &utils.F{Data:data, T:&testing.T{}} + FuzzFunction(fuzzerF) + } + if profname != "" { + pprof.StopCPUProfile() + f, err := os.Create(profname + ".heap.prof") + if err != nil { + t.Logf("error creating heap profile file %s\n", err) + } + if err = pprof.WriteHeapProfile(f); err != nil { + t.Logf("error writing heap profile file %s\n", err) + } + f.Close() + } +}