diff --git a/embuilder.py b/embuilder.py index 3fca4314a5703..1aab1d1316023 100755 --- a/embuilder.py +++ b/embuilder.py @@ -237,7 +237,7 @@ def main(): if do_clear: library.erase() if do_build: - library.get_path() + library.build() elif what == 'sysroot': if do_clear: shared.Cache.erase_file('sysroot_install.stamp') diff --git a/tools/cache.py b/tools/cache.py index f249c76276645..c09de6fe1ad6d 100644 --- a/tools/cache.py +++ b/tools/cache.py @@ -108,8 +108,8 @@ def get_lib_dir(self, absolute, varies=True): path = Path(path, '-'.join(subdir)) return path - def get_lib_name(self, name, varies=True): - return str(self.get_lib_dir(absolute=False, varies=varies).joinpath(name)) + def get_lib_name(self, name, varies=True, absolute=False): + return str(self.get_lib_dir(absolute=absolute, varies=varies).joinpath(name)) def erase_lib(self, name): self.erase_file(self.get_lib_name(name)) @@ -127,7 +127,7 @@ def get_lib(self, libname, *args, **kwargs): # Request a cached file. If it isn't in the cache, it will be created with # the given creator function - def get(self, shortname, creator, what=None, force=False): + def get(self, shortname, creator, what=None, force=False, quiet=False): cachename = Path(self.dirname, shortname) # Check for existence before taking the lock in case we can avoid the # lock completely. @@ -147,11 +147,12 @@ def get(self, shortname, creator, what=None, force=False): what = 'system library' else: what = 'system asset' - message = f'generating {what}: {shortname}... (this will be cached in "{cachename}" for subsequent builds)' + message = f'generating {what}: "{cachename}"' logger.info(message) utils.safe_ensure_dirs(cachename.parent) creator(str(cachename)) assert cachename.exists() - logger.info(' - ok') + if not quiet: + logger.info(' - ok') return str(cachename) diff --git a/tools/gen_struct_info.py b/tools/gen_struct_info.py index a2103f8674e5a..c40fe2b226b5c 100755 --- a/tools/gen_struct_info.py +++ b/tools/gen_struct_info.py @@ -246,7 +246,7 @@ def inspect_headers(headers, cflags): # TODO(sbc): If we can remove EM_EXCLUSIVE_CACHE_ACCESS then this would not longer be needed. shared.check_sanity() - compiler_rt = system_libs.Library.get_usable_variations()['libcompiler_rt'].get_path() + compiler_rt = system_libs.Library.get_usable_variations()['libcompiler_rt'].build() # Close all unneeded FDs. os.close(src_file[0]) diff --git a/tools/ports/__init__.py b/tools/ports/__init__.py index 17938a49c133f..289edd49d683c 100644 --- a/tools/ports/__init__.py +++ b/tools/ports/__init__.py @@ -129,16 +129,27 @@ def build_port(src_dir, output_path, build_dir, includes=[], flags=[], exclude_f if not os.path.exists(build_dir): os.makedirs(build_dir) build_dir = src_dir - commands = [] - objects = [] - for src in srcs: - relpath = os.path.relpath(src, src_dir) - obj = os.path.join(build_dir, relpath) + '.o' - commands.append([shared.EMCC, '-c', src, '-o', obj] + cflags) - objects.append(obj) - - system_libs.run_build_commands(commands) - system_libs.create_lib(output_path, objects) + + if system_libs.USE_NINJA: + ninja_file = os.path.join(build_dir, 'build.ninja') + system_libs.ensure_sysroot() + system_libs.create_ninja_file(srcs, ninja_file, output_path, cflags=cflags) + cmd = ['ninja', '-C', build_dir] + if shared.PRINT_STAGES: + cmd.append('-v') + shared.check_call(cmd, env=system_libs.clean_env()) + else: + commands = [] + objects = [] + for src in srcs: + relpath = os.path.relpath(src, src_dir) + obj = os.path.join(build_dir, relpath) + '.o' + commands.append([shared.EMCC, '-c', src, '-o', obj] + cflags) + objects.append(obj) + + system_libs.run_build_commands(commands) + system_libs.create_lib(output_path, objects) + return output_path @staticmethod @@ -271,9 +282,16 @@ def clear_project_build(name): port = ports_by_name[name] port.clear(Ports, settings, shared) build_dir = os.path.join(Ports.get_build_dir(), name) - utils.delete_dir(build_dir) + if not system_libs.USE_NINJA: + utils.delete_dir(build_dir) return build_dir + @staticmethod + def write_file(filename, contents): + if os.path.exists(filename) and utils.read_file(filename) == contents: + return + utils.write_file(filename, contents) + def dependency_order(port_list): # Perform topological sort of ports according to the dependency DAG diff --git a/tools/ports/boost_headers.py b/tools/ports/boost_headers.py index 36b07f4f19aff..d113b23245491 100644 --- a/tools/ports/boost_headers.py +++ b/tools/ports/boost_headers.py @@ -5,7 +5,6 @@ import logging import os -from pathlib import Path TAG = '1.75.0' HASH = '8c38be1ebef1b8ada358ad6b7c9ec17f5e0a300e8085db3473a13e19712c95eeb3c3defacd3c53482eb96368987c4b022efa8da2aac2431a154e40153d3c3dcd' @@ -31,7 +30,7 @@ def create(final): build_dir = ports.clear_project_build('boost_headers') dummy_file = os.path.join(build_dir, 'dummy.cpp') shared.safe_ensure_dirs(os.path.dirname(dummy_file)) - Path(dummy_file).write_text('static void dummy() {}') + ports.write_file(dummy_file, 'static void dummy() {}') ports.build_port(build_dir, final, build_dir) diff --git a/tools/ports/freetype.py b/tools/ports/freetype.py index e75f9018c7ea9..9b6bf548b7e46 100644 --- a/tools/ports/freetype.py +++ b/tools/ports/freetype.py @@ -19,7 +19,7 @@ def get(ports, settings, shared): def create(final): source_path = os.path.join(ports.get_dir(), 'freetype', 'FreeType-' + TAG) - Path(source_path, 'include/ftconfig.h').write_text(ftconf_h) + ports.write_file(Path(source_path, 'include/ftconfig.h'), ftconf_h) ports.install_header_dir(os.path.join(source_path, 'include'), target=os.path.join('freetype2')) diff --git a/tools/ports/libjpeg.py b/tools/ports/libjpeg.py index 8c52e107b9c87..149d8571b2e50 100644 --- a/tools/ports/libjpeg.py +++ b/tools/ports/libjpeg.py @@ -25,7 +25,7 @@ def create(final): logging.info('building port: libjpeg') source_path = os.path.join(ports.get_dir(), 'libjpeg', 'jpeg-9c') dest_path = ports.clear_project_build('libjpeg') - Path(source_path, 'jconfig.h').write_text(jconfig_h) + ports.write_file(Path(source_path, 'jconfig.h'), jconfig_h) ports.install_headers(source_path) excludes = [ 'ansi2knr.c', 'cjpeg.c', 'ckconfig.c', 'djpeg.c', 'example.c', diff --git a/tools/ports/libmodplug.py b/tools/ports/libmodplug.py index d8343babe6ee1..0d99646e45147 100644 --- a/tools/ports/libmodplug.py +++ b/tools/ports/libmodplug.py @@ -25,7 +25,7 @@ def create(final): src_dir = os.path.join(source_path, 'src') libmodplug_path = os.path.join(src_dir, 'libmodplug') - Path(source_path, 'config.h').write_text(config_h) + ports.write_file(Path(source_path, 'config.h'), config_h) flags = [ '-Wno-deprecated-register', diff --git a/tools/ports/libpng.py b/tools/ports/libpng.py index ea32d42e6b236..a329ad845fafb 100644 --- a/tools/ports/libpng.py +++ b/tools/ports/libpng.py @@ -30,7 +30,7 @@ def create(final): logging.info('building port: libpng') source_path = os.path.join(ports.get_dir(), 'libpng', 'libpng-' + TAG) - Path(source_path, 'pnglibconf.h').write_text(pnglibconf_h) + ports.write_file(Path(source_path, 'pnglibconf.h'), pnglibconf_h) ports.install_headers(source_path) flags = ['-sUSE_ZLIB=1'] diff --git a/tools/ports/mpg123.py b/tools/ports/mpg123.py index 9a7d15797fd20..e75d406d1c520 100644 --- a/tools/ports/mpg123.py +++ b/tools/ports/mpg123.py @@ -27,8 +27,8 @@ def create(final): libmpg123_path = os.path.join(src_path, 'libmpg123') compat_path = os.path.join(src_path, 'compat') - Path(src_path, 'config.h').write_text(config_h) - Path(libmpg123_path, 'mpg123.h').write_text(mpg123_h) + ports.write_file(Path(src_path, 'config.h'), config_h) + ports.write_file(Path(libmpg123_path, 'mpg123.h'), mpg123_h) # copy header to a location so it can be used as 'MPG123/' ports.install_headers(libmpg123_path, pattern="*123.h", target='') diff --git a/tools/ports/ogg.py b/tools/ports/ogg.py index df90552e12fb6..f27e61cd3b810 100644 --- a/tools/ports/ogg.py +++ b/tools/ports/ogg.py @@ -22,7 +22,7 @@ def create(final): logging.info('building port: ogg') source_path = os.path.join(ports.get_dir(), 'ogg', 'Ogg-' + TAG) - Path(source_path, 'include', 'ogg', 'config_types.h').write_text(config_types_h) + ports.write_file(Path(source_path, 'include', 'ogg', 'config_types.h'), config_types_h) ports.install_header_dir(os.path.join(source_path, 'include', 'ogg'), 'ogg') dest_path = ports.clear_project_build('ogg') ports.build_port(os.path.join(source_path, 'src'), final, dest_path) diff --git a/tools/ports/zlib.py b/tools/ports/zlib.py index a53d07ed019f7..7307c0862db93 100644 --- a/tools/ports/zlib.py +++ b/tools/ports/zlib.py @@ -4,7 +4,6 @@ # found in the LICENSE file. import os -from pathlib import Path VERSION = '1.2.12' HASH = 'cc2366fa45d5dfee1f983c8c51515e0cff959b61471e2e8d24350dea22d3f6fcc50723615a911b046ffc95f51ba337d39ae402131a55e6d1541d3b095d6c0a14' @@ -19,7 +18,7 @@ def get(ports, settings, shared): def create(final): source_path = os.path.join(ports.get_dir(), 'zlib', 'zlib-' + VERSION) - Path(source_path, 'zconf.h').write_text(zconf_h) + ports.write_file(os.path.join(source_path, 'zconf.h'), zconf_h) ports.install_headers(source_path) # build diff --git a/tools/system_libs.py b/tools/system_libs.py index d7ebcb2acc35c..5d038b1c09a8e 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -28,6 +28,14 @@ 'getsockopt.c', 'setsockopt.c', 'freeaddrinfo.c', 'in6addr_any.c', 'in6addr_loopback.c', 'accept4.c'] +# Experimental: Setting EMCC_USE_NINJA will cause system libraries to get built with ninja rather +# than simple subprocesses. The primary benefit here is that we get accurate dependency tracking. +# This means we can avoid completely rebuilding a library and just rebuild based on what changed. +# +# Setting EMCC_USE_NINJA=2 means that ninja will automatically be run for each library needed at +# link time. +USE_NINJA = int(os.environ.get('EMCC_USE_NINJA', '0')) + def files_in_path(path, filenames): srcdir = utils.path_from_root(path) @@ -93,6 +101,107 @@ def create_lib(libname, inputs): building.emar('cr', libname, inputs) +def create_ninja_file(input_files, filename, libname, cflags, asflags=None, customize_build_flags=None): + if asflags is None: + asflags = [] + # TODO(sbc) There is an llvm bug that causes a crash when `-g` is used with + # assembly files that define wasm globals. + asflags = [arg for arg in asflags if arg != '-g'] + cflags_asm = [arg for arg in cflags if arg != '-g'] + + def join(flags): + return ' '.join(flags) + + out = f'''\ +# Automatically generated by tools/system_libs.py. DO NOT EDIT + +ninja_required_version = 1.5 + +ASFLAGS = {join(asflags)} +CFLAGS = {join(cflags)} +CFLAGS_ASM = {join(cflags_asm)} +EMCC = {shared.EMCC} +EMXX = {shared.EMXX} +EMAR = {shared.EMAR} + +rule cc + depfile = $out.d + command = $EMCC -MD -MF $out.d $CFLAGS -c $in -o $out + description = CC $out + +rule cxx + depfile = $out.d + command = $EMXX -MD -MF $out.d $CFLAGS -c $in -o $out + description = CXX $out + +rule asm + command = $EMCC $ASFLAGS -c $in -o $out + description = ASM $out + +rule asm_cpp + depfile = $out.d + command = $EMCC -MD -MF $out.d $CFLAGS_ASM -c $in -o $out + description = ASM $out + +rule direct_cc + depfile = $with_depfile + command = $EMCC -MD -MF $with_depfile $CFLAGS -c $in -o $out + description = CC $out + +rule archive + command = $EMAR cr $out $in + description = AR $out + +''' + suffix = shared.suffix(libname) + + case_insensitive = is_case_insensitive(os.path.dirname(filename)) + if suffix == '.o': + assert len(input_files) == 1 + depfile = shared.unsuffixed_basename(input_files[0]) + '.d' + out += f'build {libname}: direct_cc {input_files[0]}\n' + out += f' with_depfile = {depfile}\n' + else: + objects = [] + for src in input_files: + # Resolve duplicates by appending unique. + # This is needed on case insensitve filesystem to handle, + # for example, _exit.o and _Exit.o. + o = shared.unsuffixed_basename(src) + '.o' + object_uuid = 0 + if case_insensitive: + o = o.lower() + # Find a unique basename + while o in objects: + object_uuid += 1 + o = f'{o}__{object_uuid}.o' + objects.append(o) + ext = shared.suffix(src) + if ext == '.s': + out += f'build {o}: asm {src}\n' + flags = asflags + elif ext == '.S': + out += f'build {o}: asm_cpp {src}\n' + flags = cflags_asm + elif ext == '.c': + out += f'build {o}: cc {src}\n' + flags = cflags + else: + out += f'build {o}: cxx {src}\n' + flags = cflags + if customize_build_flags: + custom_flags = customize_build_flags(flags, src) + if custom_flags != flags: + out += f' CFLAGS = {join(custom_flags)}' + out += '\n' + + objects = sorted(objects, key=lambda x: os.path.basename(x)) + objects = ' '.join(objects) + out += f'build {libname}: archive {objects}\n' + + utils.write_file(filename, out) + + def is_case_insensitive(path): """Returns True if the filesystem at `path` is case insensitive.""" utils.write_file(os.path.join(path, 'test_file'), '') @@ -240,15 +349,18 @@ def can_build(self): return True def erase(self): - shared.Cache.erase_file(shared.Cache.get_lib_name(self.get_filename())) + shared.Cache.erase_file(self.get_path()) - def get_path(self): + def get_path(self, absolute=False): + return shared.Cache.get_lib_name(self.get_filename(), absolute=absolute) + + def build(self): """ Gets the cached path of this library. This will trigger a build if this library is not in the cache. """ - return shared.Cache.get_lib(self.get_filename(), self.build) + return shared.Cache.get(self.get_path(), self.do_build, force=USE_NINJA == 2, quiet=USE_NINJA) def get_link_flag(self): """ @@ -256,7 +368,7 @@ def get_link_flag(self): This will trigger a build if this library is not in the cache. """ - fullpath = self.get_path() + fullpath = self.build() # For non-libaries (e.g. crt1.o) we pass the entire path to the linker if self.get_ext() != '.a': return fullpath @@ -282,6 +394,12 @@ def get_files(self): raise NotImplementedError() + def write_ninja_file(self, filename, libname): + cflags = self.get_cflags() + asflags = get_base_cflags() + input_files = self.get_files() + create_ninja_file(input_files, filename, libname, cflags, asflags=asflags, customize_build_flags=self.customize_build_cmd) + def build_objects(self, build_dir): """ Returns a list of compiled object files for this library. @@ -335,13 +453,27 @@ def customize_build_cmd(self, cmd, filename): # noqa For example, libc uses this to replace -Oz with -O2 for some subset of files.""" return cmd - def build(self, out_filename): + def do_build(self, out_filename): """Builds the library and returns the path to the file.""" - build_dir = shared.Cache.get_path(os.path.join('build', self.get_base_name())) - utils.safe_ensure_dirs(build_dir) - create_lib(out_filename, self.build_objects(build_dir)) - if not shared.DEBUG: - utils.delete_dir(build_dir) + assert out_filename == self.get_path(absolute=True) + build_dir = os.path.join(shared.Cache.get_path('build'), self.get_base_name()) + if USE_NINJA: + ensure_sysroot() + utils.safe_ensure_dirs(build_dir) + ninja_file = os.path.join(build_dir, 'build.ninja') + self.write_ninja_file(ninja_file, out_filename) + cmd = ['ninja', '-C', build_dir] + if shared.PRINT_STAGES: + cmd.append('-v') + shared.check_call(cmd, env=clean_env()) + else: + # Use a seperate build directory to the ninja flavor so that building without + # EMCC_USE_NINJA doesn't clobber the ninja build tree + build_dir += '-tmp' + utils.safe_ensure_dirs(build_dir) + create_lib(out_filename, self.build_objects(build_dir)) + if not shared.DEBUG: + utils.delete_dir(build_dir) @classmethod def _inherit_list(cls, attr):