From eef03e0cc66614fe4f7de55257daa26c739c3a4f Mon Sep 17 00:00:00 2001 From: jonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com> Date: Tue, 30 Aug 2022 16:28:37 -0400 Subject: [PATCH] [aarch64] Add ARM support to helper.py (#8332) This is probably a lot of what is needed for CFL support (though there's probably a lot of other stuff left). Related: #8164 --- infra/helper.py | 69 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/infra/helper.py b/infra/helper.py index 988a93874e02..19b25772a604 100755 --- a/infra/helper.py +++ b/infra/helper.py @@ -69,6 +69,7 @@ WORKDIR_REGEX = re.compile(r'\s*WORKDIR\s*([^\s]+)') LANGUAGES_WITH_BUILDER_IMAGES = {'go', 'jvm', 'python', 'rust', 'swift'} +ARM_BUILDER_NAME = 'oss-fuzz-buildx-builder' if sys.version_info[0] >= 3: raw_input = input # pylint: disable=invalid-name @@ -240,6 +241,7 @@ def get_parser(): # pylint: disable=too-many-statements build_image_parser.add_argument('--pull', action='store_true', help='Pull latest base image.') + _add_architecture_args(build_image_parser) build_image_parser.add_argument('--cache', action='store_true', default=False, @@ -387,9 +389,10 @@ def check_project_exists(project): return False -def _check_fuzzer_exists(project, fuzzer_name): +def _check_fuzzer_exists(project, fuzzer_name, architecture='x86_64'): """Checks if a fuzzer exists.""" - command = ['docker', 'run', '--rm'] + platform = 'linux/arm64' if architecture == 'aarch64' else 'linux/amd64' + command = ['docker', 'run', '--rm', '--platform', platform] command.extend(['-v', '%s:/out' % project.out]) command.append(BASE_RUNNER_IMAGE) @@ -466,7 +469,7 @@ def _add_environment_args(parser): help="set environment variable e.g. VAR=value") -def build_image_impl(project, cache=True, pull=False): +def build_image_impl(project, cache=True, pull=False, architecture='x86_64'): """Builds image.""" image_name = project.name @@ -486,14 +489,27 @@ def build_image_impl(project, cache=True, pull=False): return False build_args = [] + image_name = 'gcr.io/%s/%s' % (image_project, image_name) + if architecture == 'aarch64': + build_args += [ + 'buildx', + 'build', + '--platform', + 'linux/arm64', + '--progress', + 'plain', + '--load', + ] if not cache: build_args.append('--no-cache') - build_args += [ - '-t', - 'gcr.io/%s/%s' % (image_project, image_name), '--file', dockerfile_path - ] + build_args += ['-t', image_name, '--file', dockerfile_path] build_args.append(docker_build_dir) + + if architecture == 'aarch64': + command = ['docker'] + build_args + subprocess.check_call(command) + return True return docker_build(build_args) @@ -526,10 +542,18 @@ def _workdir_from_dockerfile(project): return workdir_from_lines(lines, default=os.path.join('/src', project.name)) -def docker_run(run_args, print_output=True): - """Calls `docker run`.""" - command = ['docker', 'run', '--rm', '--privileged'] +def prepare_aarch64_emulation(): + """Run some necessary commands to use buildx to build AArch64 targets using + QEMU emulation on an x86_64 host.""" + subprocess.check_call( + ['docker', 'buildx', 'create', '--name', ARM_BUILDER_NAME]) + subprocess.check_call(['docker', 'buildx', 'use', ARM_BUILDER_NAME]) + +def docker_run(run_args, print_output=True, architecture='x86_64'): + """Calls `docker run`.""" + platform = 'linux/arm64' if architecture == 'aarch64' else 'linux/amd64' + command = ['docker', 'run', '--rm', '--privileged', '--platform', platform] # Support environments with a TTY. if sys.stdin.isatty(): command.append('-i') @@ -598,7 +622,10 @@ def build_image(args): logging.info('Using cached base images...') # If build_image is called explicitly, don't use cache. - if build_image_impl(args.project, cache=args.cache, pull=pull): + if build_image_impl(args.project, + cache=args.cache, + pull=pull, + architecture=args.architecture): return True return False @@ -614,7 +641,7 @@ def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,to source_path, mount_path=None): """Builds fuzzers.""" - if not build_image_impl(project): + if not build_image_impl(project, architecture=architecture): return False if clean: @@ -625,13 +652,15 @@ def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,to '-v', '%s:/out' % project.out, '-t', 'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /out/*' - ]) + ], + architecture=architecture) docker_run([ '-v', '%s:/work' % project.work, '-t', 'gcr.io/oss-fuzz/%s' % project.name, '/bin/bash', '-c', 'rm -rf /work/*' - ]) + ], + architecture=architecture) else: logging.info('Keeping existing build artifacts as-is (if any).') @@ -673,7 +702,7 @@ def build_fuzzers_impl( # pylint: disable=too-many-arguments,too-many-locals,to 'gcr.io/oss-fuzz/%s' % project.name ] - result = docker_run(command) + result = docker_run(command, architecture=architecture) if not result: logging.error('Building fuzzers failed.') return False @@ -705,8 +734,8 @@ def check_build(args): if not check_project_exists(args.project): return False - if (args.fuzzer_name and - not _check_fuzzer_exists(args.project, args.fuzzer_name)): + if (args.fuzzer_name and not _check_fuzzer_exists( + args.project, args.fuzzer_name, args.architecture)): return False env = [ @@ -728,7 +757,7 @@ def check_build(args): else: run_args.append('test_all.py') - result = docker_run(run_args) + result = docker_run(run_args, architecture=args.architecture) if result: logging.info('Check build passed.') else: @@ -940,7 +969,7 @@ def run_fuzzer(args): args.fuzzer_name, ] + args.fuzzer_args) - return docker_run(run_args) + return docker_run(run_args, architecture=args.architecture) def reproduce(args): @@ -1128,7 +1157,7 @@ def shell(args): 'gcr.io/%s/%s' % (image_project, args.project.name), '/bin/bash' ]) - docker_run(run_args) + docker_run(run_args, architecture=args.architecture) return True