From 80dd8b61297e31c1e021a80ee86703f5b5188c0a Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Fri, 25 Nov 2022 17:58:57 +0000 Subject: [PATCH] fluxcd: Add auto discovery for Go Native fuzz targets (#9064) Add support for auto discovery and build for all Go Native fuzz targets that exist within the range of Flux org repos defined. Support for faster CI checks by only targeting the specific project when running against PRs. (cc: @stefanprodan @hiddeco) --- Would there be interest on having `loop_through_org_repositories` as part of infra? This would enable Go Native projects to have auto discovery of fuzzers without the need of calling `compile_native_go_fuzzer` for each fuzzer they add into their repos. Signed-off-by: Paulo Gomes --- projects/fluxcd/Dockerfile | 12 +++- projects/fluxcd/build.sh | 110 ++++++++++++++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 6 deletions(-) diff --git a/projects/fluxcd/Dockerfile b/projects/fluxcd/Dockerfile index 2a9e357c1749..aa8832cca1f9 100644 --- a/projects/fluxcd/Dockerfile +++ b/projects/fluxcd/Dockerfile @@ -16,14 +16,20 @@ FROM gcr.io/oss-fuzz-base/base-builder-go -ENV PROJECT_ROOT="${GOPATH:-/root/go}/src/github.com/fluxcd" +# cmake and pkg-config are only needed whilst libgit2 is in use. +# The project aims to deprecate libgit2 at some point in Q1 2023, +# by which point this can be removed. +RUN apt-get update && apt-get install -y cmake pkg-config -RUN mkdir -p "${PROJECT_ROOT}" +ENV GOPATH="${GOPATH:-/root/go}" +ENV ORG_ROOT="${ORG_ROOT:-${GOPATH}/src/github.com/fluxcd}" + +RUN mkdir -p "${ORG_ROOT}" # Flux has its components scattered around multiple repositories due to its architecture. # Here we clone all of them. The build process happens as build.sh iterate over each one of them. ARG REPOSITORIES="pkg notification-controller kustomize-controller helm-controller image-reflector-controller source-controller image-automation-controller" -RUN for repo in ${REPOSITORIES}; do git clone --depth 1 "https://github.com/fluxcd/${repo}" "${PROJECT_ROOT}/${repo}"; done +RUN for repo in ${REPOSITORIES}; do git clone --depth 1 "https://github.com/fluxcd/${repo}" "${ORG_ROOT}/${repo}"; done COPY build.sh $SRC/ WORKDIR $SRC diff --git a/projects/fluxcd/build.sh b/projects/fluxcd/build.sh index 3cfa631cd996..41e7412ee868 100644 --- a/projects/fluxcd/build.sh +++ b/projects/fluxcd/build.sh @@ -15,7 +15,111 @@ # ################################################################################ -PROJECT_ROOT="${GOPATH:-/root/go}/src/github.com/fluxcd" +# This code improves the use of Go Native by: +# - Dynamically discovering and building all fuzz tests within the project root path. +# - Supporting single (during PR checks) or multiple repositories (oss-fuzz). +# - Enabling execution via CI builds and Makefile targets for each repo. -# Build the fuzzers for all cloned sub-projects -find "${PROJECT_ROOT}/" -type f -name oss_fuzz_build.sh | bash -e +GOPATH="${GOPATH:-/root/go}" +ORG_ROOT="${ORG_ROOT:-${GOPATH}/src/github.com/fluxcd}" +PREBUILD_SCRIPT_PATH="${PREBUILD_SCRIPT_PATH:-tests/fuzz/oss_fuzz_prebuild.sh}" +POSTBUILD_SCRIPT_PATH="${POSTBUILD_SCRIPT_PATH:-tests/fuzz/oss_fuzz_postbuild.sh}" +FLUX_CI="${FLUX_CI:-false}" + +# source_prebuild_script sources the prebuild script, which executes project-specific +# code and exposes environment variables that are needed during the generic build process. +# +# Examples of usage may be organising directory structure for embedding +# files, downloading artifacts or setting environment variables. +function source_prebuild_script(){ + if [ -f "${PREBUILD_SCRIPT_PATH}" ]; then + # shellcheck source=/dev/null + . "${PREBUILD_SCRIPT_PATH}" + fi +} + +# source_postbuild_script sources the postbuild script, which executes project-specific +# code and unset environment variables that may break follow-up processes. +function source_postbuild_script(){ + if [ -f "${POSTBUILD_SCRIPT_PATH}" ]; then + # shellcheck source=/dev/null + . "${POSTBUILD_SCRIPT_PATH}" + fi +} + +# go_native_build_all_fuzzers builds all Go Native fuzz tests defined in modules within +# the given project dir. +# +# Args: +# project_dir +function go_native_build_all_fuzzers(){ + local project_path="$1" + + cd "${project_path}" + + source_prebuild_script + + modules=$(find . -mindepth 1 -maxdepth 4 -type f -name 'go.mod' | cut -c 3- | sed 's|/[^/]*$$||' | sort -u | sed 's;/go.mod;;g' | sed 's;go.mod;.;g') + for module in ${modules}; do + cd "${project_path}/${module}" + + local test_files + test_files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' . || echo "") + if [ -z "${test_files}" ]; then + continue + fi + + # go-118-fuzz-build is required for each module. + go get github.com/AdamKorcz/go-118-fuzz-build/testing + + # Iterate through all Go Fuzz targets, compiling each into a fuzzer. + for file in ${test_files}; do + # If the subdir is a module, skip this file, as it will be handled + # at the next iteration of the outer loop. + if [ -f "$(dirname "${file}")/go.mod" ]; then + continue + fi + + targets=$(grep -oP 'func \K(Fuzz\w*)' "${file}") + for target_name in ${targets}; do + local module_name + local fuzzer_name + local target_dir + + # Transform module path into module name (e.g. git/libgit2 to git_libgit2). + module_name="${module/\//_}_" + # If module equal '._', use empty string instead. + module_name="${module/#%._}" + + # Compose fuzzer name based on the lowercase version of the func names. + fuzzer_name="${target_name,,}" + # The module name is added after the fuzz prefix, for better discoverability. + fuzzer_name="${target_name/fuzz_/fuzz_${module_name}}" + target_dir=$(dirname "${file}") + + echo "Building ${file}.${target_name} into ${fuzzer_name}" + compile_native_go_fuzzer "${target_dir}" "${target_name}" "${fuzzer_name}" + done + done + done +} + +function loop_through_org_repositories(){ + local repos="" + repos="$(find "${ORG_ROOT}" -type d -mindepth 1 -maxdepth 1)" + for repo in ${repos}; do + go_native_build_all_fuzzers "${repo}" + done +} + +function main(){ + if [[ "${FLUX_CI}" == "true" ]]; then + echo "Building Go Native fuzzers for Flux CI" + go_native_build_all_fuzzers "${SRC}" + else + echo "Going through all repositories in ${ORG_ROOT}" + loop_through_org_repositories + fi +} + +main