[infra] do not checkout oss-fuzz (#23)

Promising oss-fuzz in /src/oss-fuzz creates lots of confusion about where files come from. Let's make everything explicit. Fixes #20
google · Oct 12, 2016 · dae2012 · dae2012
1 parent 1eddcd9
commit dae2012
Show file tree

Hide file tree

Showing 25 changed files with 60 additions and 52 deletions.
diff --git a/docs/building_running_fuzzers_external.md b/docs/building_running_fuzzers_external.md
@@ -29,7 +29,7 @@ docker build -t ossfuzz/$PROJECT_NAME oss-fuzz/$PROJECT_NAME
 ````
 2. Running a container:
 ````bash
-docker run -ti -v $PWD/$PROJECT_NAME:/src/$PROJECT_NAME -v $PWD/oss-fuzz:/src/oss-fuzz -v /tmp/out:/out ossfuzz/$PROJECT_NAME
+docker run -ti -v $PWD/$PROJECT_NAME:/src/$PROJECT_NAME -v /tmp/out:/out ossfuzz/$PROJECT_NAME
 ````
 
 `/tmp/out` will contain fuzzers.

diff --git a/docs/new_library.md b/docs/new_library.md
@@ -43,14 +43,38 @@ Create a fuzzer and add it to the *library_name/* directory as well.
 
 This is the Docker image definition that build.sh will be executed in.
 It is very simple for most libraries:
-```bash
+```docker
 FROM ossfuzz/base-libfuzzer             # base image with clang toolchain
-MAINTAINER YOUR_EMAIL                   # each file should have a maintainer 
+MAINTAINER YOUR_EMAIL                   # each file should have a maintainer
 RUN apt-get install -y ...              # install required packages to build a project
 COPY build.sh /src/                     # install build script for the project.
 ```
 Expat example: [expat/Dockerfile](../expat/Dockerfile)
 
+## Create Fuzzer Source File
+
+Create a new .cc file, define a `LLVMFuzzerTestOneInput` function and call
+your library:
+
+```c++
+#include <stddef.h>
+#include <stdint.h>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+  // put your fuzzing code here and use data+size as input.
+  return 0;
+}
+```
+
+Make sure you add the file to your Docker image:
+```docker
+COPY build.sh my_fuzzer.cc /src/         # install build script & fuzzer.
+```
+
+There are [lots](../libxml2/libxml2_xml_read_memory_fuzzer.cc)
+[of](../expat/parse_fuzzer.cc) [examples](../zlib/zlib_uncompress_fuzzer.cc)
+in this project repository.
+
 ## build.sh
 
 This is where most of the work is done to build fuzzers for your library. The script will
@@ -105,24 +129,6 @@ These flags are provided in following environment variables:
 Many well-crafted build scripts will automatically use these variables. If not,
 passing them manually to a build tool might be required.
 
-## Create Fuzzer Source File
-
-Create a new .cc file, define a `LLVMFuzzerTestOneInput` function and call
-your library:
-
-```c++
-#include <stddef.h>
-#include <stdint.h>
-
-extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
-  // put your fuzzing code here and use data+size as input.
-  return 0;
-}
-```
-
-There are [lots](../libxml2/libxml2_xml_read_memory_fuzzer.cc) 
-[of](../expat/parse_fuzzer.cc) [examples](../zlib/zlib_uncompress_fuzzer.cc)
-in this project repository.
 
 ### Dictionaries and custom libfuzzer options
 

diff --git a/expat/Dockerfile b/expat/Dockerfile
@@ -21,4 +21,4 @@ RUN apt-get install -y make autoconf automake libtool docbook2x
 ENV GIT_CHECKOUT_DIR="expat"
 ENV GIT_URL="git://git.code.sf.net/p/expat/code_git"
 
-COPY build.sh /src/
+COPY build.sh parse_fuzzer.*  xml.dict /src/
diff --git a/expat/README.md b/expat/README.md
@@ -10,7 +10,7 @@ git clone https://github.com/google/oss-fuzz.git
 git clone git://git.code.sf.net/p/expat/code_git expat
 # Build & run the image.
 docker build -t ossfuzz/expat oss-fizz/expat && \
-docker run -i -v $PWD/oss-fuzz:/src/oss-fuzz -v $PWD/expat:/src/expat -v $HOME/tmp/out:/out -t ossfuzz/expat
+docker run -i -v $PWD/expat:/src/expat -v $HOME/tmp/out:/out -t ossfuzz/expat
 ````
 Fuzzers will be in `$HOME/tmp/out`.
 

diff --git a/expat/build.sh b/expat/build.sh
@@ -6,5 +6,7 @@ cd /src/expat/expat
 make clean all
 
 $CXX $CXXFLAGS -std=c++11 -Ilib/ \
-    /src/oss-fuzz/expat/parse_fuzzer.cc -o /out/expat_parse_fuzzer \
+    /src/parse_fuzzer.cc -o /out/expat_parse_fuzzer \
     /work/libfuzzer/*.o .libs/libexpat.a $LDFLAGS
+
+cp /src/*.dict /src/*.options /out/
diff --git a/infra/base-images/base-libfuzzer/Dockerfile b/infra/base-images/base-libfuzzer/Dockerfile
@@ -18,8 +18,6 @@ FROM ossfuzz/base-clang
 MAINTAINER [email protected]
 RUN apt-get install -y git libc6-dev
 
-RUN cd /src && git clone --depth 1 https://github.com/google/oss-fuzz.git
-
 RUN mkdir -p /work/libfuzzer
 
 ENV SANITIZER_FLAGS="-fsanitize=address"

diff --git a/infra/libfuzzer-pipeline.groovy b/infra/libfuzzer-pipeline.groovy
@@ -34,7 +34,6 @@ def call(body) {
 
     def date = java.time.format.DateTimeFormatter.ofPattern("yyyyMMddHHmm")
         .format(java.time.LocalDateTime.now())
-    def ossFuzzUrl = 'https://github.com/google/oss-fuzz.git'
 
     node {
       def workspace = pwd()
@@ -44,10 +43,6 @@ def call(body) {
 
       stage("docker image") {
           def revisions = [:]
-          dir('oss-fuzz') {
-              git url: ossFuzzUrl
-          }
-
           dir(checkoutDir) {
               git url: gitUrl
               revisions[gitUrl] = sh(returnStdout: true, script: 'git rev-parse HEAD').trim()
@@ -74,11 +69,7 @@ def call(body) {
             // Run image to produce fuzzers
             sh "rm -rf $out"
             sh "mkdir -p $out"
-            sh "docker run -v $workspace/$checkoutDir:/src/$checkoutDir -v $workspace/oss-fuzz:/src/oss-fuzz -v $out:/out -e SANITIZER_FLAGS=\"-fsanitize=$sanitizer\" -t $dockerTag"
-
-            // Copy dict and options files
-            sh "cp $workspace/oss-fuzz/$projectName/*.dict $out/ || true"
-            sh "cp $workspace/oss-fuzz/$projectName/*.options $out/ || true"
+            sh "docker run -v $workspace/$checkoutDir:/src/$checkoutDir -v $out:/out -e SANITIZER_FLAGS=\"-fsanitize=$sanitizer\" -t $dockerTag"
           }
         }
       }

diff --git a/libchewing/Dockerfile b/libchewing/Dockerfile
@@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER [email protected]
 RUN apt-get install -y make autoconf automake libtool texinfo
 
-COPY build.sh /src/
+COPY build.sh chewing_fuzzer.c /src/
diff --git a/libchewing/build.sh b/libchewing/build.sh
@@ -27,7 +27,7 @@ make -C test CFLAGS="$CFLAGS -Dmain=stress_main -Drand=get_fuzz_input" stress.o
 
 $CC $CFLAGS \
     -o /out/chewing_fuzzer \
-    /src/oss-fuzz/libchewing/chewing_fuzzer.c \
+    /src/chewing_fuzzer.c \
     test/stress.o test/.libs/libtesthelper.a src/.libs/libchewing.a \
     /work/libfuzzer/*.o $LDFLAGS
 

diff --git a/libpng/Dockerfile b/libpng/Dockerfile
@@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER [email protected]
 RUN apt-get install -y make autoconf automake libtool zlib1g-dev
 
-COPY build.sh /src/
+COPY build.sh libpng_read_fuzzer.* png.dict /src/
diff --git a/libpng/build.sh b/libpng/build.sh
@@ -29,5 +29,7 @@ make clean all
 
 # build libpng_read_fuzzer
 $CXX $CXXFLAGS -std=c++11 -I. -lz \
-     /src/oss-fuzz/libpng/libpng_read_fuzzer.cc -o /out/libpng_read_fuzzer \
+     /src/libpng_read_fuzzer.cc -o /out/libpng_read_fuzzer \
      /work/libfuzzer/*.o .libs/libpng16.a $LDFLAGS
+
+cp /src/*.dict /src/*.options /out/
diff --git a/libxml2/Dockerfile b/libxml2/Dockerfile
@@ -19,3 +19,6 @@ MAINTAINER [email protected]
 RUN apt-get install -y make autoconf automake libtool pkg-config
 
 COPY build.sh /src/
+COPY libxml2_xml_read_memory_fuzzer.* \
+     libxml2_xml_regexp_compile_fuzzer.* \
+     xml.dict /src/
diff --git a/libxml2/build.sh b/libxml2/build.sh
@@ -23,6 +23,8 @@ make clean all
 
 for fuzzer in libxml2_xml_read_memory_fuzzer libxml2_xml_regexp_compile_fuzzer; do
   $CXX $CXXFLAGS -std=c++11 -Iinclude/ \
-      /src/oss-fuzz/libxml2/$fuzzer.cc -o /out/$fuzzer \
+      /src/$fuzzer.cc -o /out/$fuzzer \
       /work/libfuzzer/*.o .libs/libxml2.a $LDFLAGS
 done
+
+cp /src/*.dict /src/*.options /out/
diff --git a/nss/Dockerfile b/nss/Dockerfile
@@ -18,6 +18,6 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER [email protected]
 RUN apt-get install -y make autoconf automake libtool mercurial zlib1g-dev
 
-COPY build.sh /src/
+COPY build.sh fuzzers /src/
 
 ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/out"
diff --git a/nss/build.sh b/nss/build.sh
@@ -57,7 +57,7 @@ FUZZERS="asn1_algorithmid_fuzzer \
 
 
 for fuzzer in $FUZZERS; do
-  $CXX $CXXFLAGS -std=c++11 /src/oss-fuzz/nss/fuzzers/$fuzzer.cc \
+  $CXX $CXXFLAGS -std=c++11 /src/fuzzers/$fuzzer.cc \
      -I/work/nss/include \
      /work/libfuzzer/*.o \
      /work/nss/lib/libnss.a /work/nss/lib/libnssutil.a \

diff --git a/re2/Dockerfile b/re2/Dockerfile
@@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER [email protected]
 RUN apt-get install -y make autoconf automake libtool
 
-COPY build.sh /src/
+COPY build.sh re2_fuzzer.* /src/
diff --git a/re2/build.sh b/re2/build.sh
@@ -31,6 +31,7 @@ make obj/libre2.a
 
 # Second, build our fuzzers.
 $CXX $CXXFLAGS -std=c++11 -I. \
-	/src/oss-fuzz/re2/re2_fuzzer.cc -o /out/re2_fuzzer \
+	/src/re2_fuzzer.cc -o /out/re2_fuzzer \
 	/work/libfuzzer/*.o ./obj/libre2.a $LDFLAGS
 
+cp /src/*.options /src/*.dict /out/\
diff --git a/re2/re2.options → re2/re2_fuzzer.options b/re2/re2.options → re2/re2_fuzzer.options
diff --git a/scripts/helper.py b/scripts/helper.py
@@ -272,7 +272,6 @@ def shell(shell_args):
 
   command = [
         'docker', 'run', '-i',
-        '-v', '%s:/src/oss-fuzz' % OSSFUZZ_DIR,
         '-v', '%s:/src/%s' % (checkout_dir, args.library_name),
         '-v', '%s:/out' % os.path.join(BUILD_DIR, 'out', args.library_name),
         '-t', 'ossfuzz/' + args.library_name,

diff --git a/sqlite3/Dockerfile b/sqlite3/Dockerfile
@@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER [email protected]
 RUN apt-get install -y make autoconf automake libtool fossil tcl
 
-COPY build.sh /src/
+COPY build.sh sqlite3_fuzzer.* sql.dict /src/
diff --git a/sqlite3/build.sh b/sqlite3/build.sh
@@ -33,5 +33,7 @@ make
 make sqlite3.c
 
 $CXX $CXXFLAGS -std=c++11 -I. \
-    /src/oss-fuzz/sqlite3/sqlite3_fuzzer.cc -o /out/sqlite3_fuzzer \
+    /src/sqlite3_fuzzer.cc -o /out/sqlite3_fuzzer \
     /work/libfuzzer/*.o ./sqlite3.o $LDFLAGS
+
+cp /src/*.options /src/*.dict /out/
diff --git a/woff2/Dockerfile b/woff2/Dockerfile
@@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER [email protected]
 RUN apt-get install -y make autoconf automake libtool
 
-COPY build.sh /src/
+COPY build.sh convert_woff2ttf_fuzzer.* /src/
diff --git a/woff2/build.sh b/woff2/build.sh
@@ -41,5 +41,7 @@ rm src/woff2_compress.o src/woff2_decompress.o
 # Build the fuzzer.
 fuzzer=convert_woff2ttf_fuzzer
 $CXX $CXXFLAGS -std=c++11 -Isrc \
-    /src/oss-fuzz/woff2/$fuzzer.cc -o /out/$fuzzer \
+    /src/$fuzzer.cc -o /out/$fuzzer \
     /work/libfuzzer/*.o src/*.o brotli/dec/*.o brotli/enc/*.o $LDFLAGS
+
+cp /src/*.options /out/
diff --git a/zlib/Dockerfile b/zlib/Dockerfile
@@ -18,4 +18,4 @@ FROM ossfuzz/base-libfuzzer
 MAINTAINER [email protected]
 RUN apt-get install -y make autoconf automake libtool
 
-COPY build.sh /src/
+COPY build.sh zlib_uncompress_fuzzer.cc /src/
diff --git a/zlib/build.sh b/zlib/build.sh
@@ -6,5 +6,5 @@ cd /src/zlib
 make clean all
 
 $CXX $CXXFLAGS -std=c++11 -I. \
-    /src/oss-fuzz/zlib/zlib_uncompress_fuzzer.cc -o /out/zlib_uncompress_fuzzer \
+    /src/zlib_uncompress_fuzzer.cc -o /out/zlib_uncompress_fuzzer \
     /work/libfuzzer/*.o ./libz.a $LDFLAGS