diff --git a/embuilder.py b/embuilder.py index 3fca4314a5703..1aab1d1316023 100755 --- a/embuilder.py +++ b/embuilder.py @@ -237,7 +237,7 @@ def main(): if do_clear: library.erase() if do_build: - library.get_path() + library.build() elif what == 'sysroot': if do_clear: shared.Cache.erase_file('sysroot_install.stamp') diff --git a/tools/cache.py b/tools/cache.py index f249c76276645..795af94cc2daa 100644 --- a/tools/cache.py +++ b/tools/cache.py @@ -108,8 +108,8 @@ def get_lib_dir(self, absolute, varies=True): path = Path(path, '-'.join(subdir)) return path - def get_lib_name(self, name, varies=True): - return str(self.get_lib_dir(absolute=False, varies=varies).joinpath(name)) + def get_lib_name(self, name, varies=True, absolute=False): + return str(self.get_lib_dir(absolute=absolute, varies=varies).joinpath(name)) def erase_lib(self, name): self.erase_file(self.get_lib_name(name)) @@ -127,7 +127,7 @@ def get_lib(self, libname, *args, **kwargs): # Request a cached file. If it isn't in the cache, it will be created with # the given creator function - def get(self, shortname, creator, what=None, force=False): + def get(self, shortname, creator, what=None, force=False, quiet=False): cachename = Path(self.dirname, shortname) # Check for existence before taking the lock in case we can avoid the # lock completely. @@ -152,6 +152,7 @@ def get(self, shortname, creator, what=None, force=False): utils.safe_ensure_dirs(cachename.parent) creator(str(cachename)) assert cachename.exists() - logger.info(' - ok') + if not quiet: + logger.info(' - ok') return str(cachename) diff --git a/tools/gen_struct_info.py b/tools/gen_struct_info.py index a2103f8674e5a..c40fe2b226b5c 100755 --- a/tools/gen_struct_info.py +++ b/tools/gen_struct_info.py @@ -246,7 +246,7 @@ def inspect_headers(headers, cflags): # TODO(sbc): If we can remove EM_EXCLUSIVE_CACHE_ACCESS then this would not longer be needed. shared.check_sanity() - compiler_rt = system_libs.Library.get_usable_variations()['libcompiler_rt'].get_path() + compiler_rt = system_libs.Library.get_usable_variations()['libcompiler_rt'].build() # Close all unneeded FDs. os.close(src_file[0]) diff --git a/tools/ports/__init__.py b/tools/ports/__init__.py index ce0dcb1073068..f6ac334abd9c4 100644 --- a/tools/ports/__init__.py +++ b/tools/ports/__init__.py @@ -126,19 +126,28 @@ def build_port(src_dir, output_path, port_name, includes=[], flags=[], exclude_f for include in includes: cflags.append('-I' + include) - commands = [] - objects = [] - for src in srcs: - relpath = os.path.relpath(src, src_dir) - obj = os.path.join(build_dir, relpath) + '.o' - dirname = os.path.dirname(obj) - if not os.path.exists(dirname): - os.makedirs(dirname) - commands.append([shared.EMCC, '-c', src, '-o', obj] + cflags) - objects.append(obj) - - system_libs.run_build_commands(commands) - system_libs.create_lib(output_path, objects) + if system_libs.USE_NINJA: + if not os.path.exists(build_dir): + os.makedirs(build_dir) + ninja_file = os.path.join(build_dir, 'build.ninja') + system_libs.ensure_sysroot() + system_libs.create_ninja_file(srcs, ninja_file, output_path, cflags=cflags) + system_libs.run_ninja(build_dir) + else: + commands = [] + objects = [] + for src in srcs: + relpath = os.path.relpath(src, src_dir) + obj = os.path.join(build_dir, relpath) + '.o' + dirname = os.path.dirname(obj) + if not os.path.exists(dirname): + os.makedirs(dirname) + commands.append([shared.EMCC, '-c', src, '-o', obj] + cflags) + objects.append(obj) + + system_libs.run_build_commands(commands) + system_libs.create_lib(output_path, objects) + return output_path @staticmethod @@ -274,6 +283,12 @@ def clear_project_build(name): utils.delete_dir(build_dir) return build_dir + @staticmethod + def write_file(filename, contents): + if os.path.exists(filename) and utils.read_file(filename) == contents: + return + utils.write_file(filename, contents) + def dependency_order(port_list): # Perform topological sort of ports according to the dependency DAG diff --git a/tools/ports/boost_headers.py b/tools/ports/boost_headers.py index 0e0eb087c9c61..5ff60a7633dc5 100644 --- a/tools/ports/boost_headers.py +++ b/tools/ports/boost_headers.py @@ -5,7 +5,6 @@ import logging import os -from pathlib import Path TAG = '1.75.0' HASH = '8c38be1ebef1b8ada358ad6b7c9ec17f5e0a300e8085db3473a13e19712c95eeb3c3defacd3c53482eb96368987c4b022efa8da2aac2431a154e40153d3c3dcd' @@ -31,7 +30,7 @@ def create(final): # this is needed as emscripted ports expect this, even if it is not used dummy_file = os.path.join(source_path, 'dummy.cpp') shared.safe_ensure_dirs(os.path.dirname(dummy_file)) - Path(dummy_file).write_text('static void dummy() {}') + ports.write_file(dummy_file, 'static void dummy() {}') ports.build_port(source_path, final, 'boost_headers', srcs=['dummy.cpp']) diff --git a/tools/ports/freetype.py b/tools/ports/freetype.py index 4fbeb9ee36275..05d9a85bcb13c 100644 --- a/tools/ports/freetype.py +++ b/tools/ports/freetype.py @@ -4,7 +4,6 @@ # found in the LICENSE file. import os -from pathlib import Path TAG = 'version_1' HASH = '0d0b1280ba0501ad0a23cf1daa1f86821c722218b59432734d3087a89acd22aabd5c3e5e1269700dcd41e87073046e906060f167c032eb91a3ac8c5808a02783' @@ -19,7 +18,7 @@ def get(ports, settings, shared): def create(final): source_path = os.path.join(ports.get_dir(), 'freetype', 'FreeType-' + TAG) - Path(source_path, 'include/ftconfig.h').write_text(ftconf_h) + ports.write_file(os.path.join(source_path, 'include/ftconfig.h'), ftconf_h) ports.install_header_dir(os.path.join(source_path, 'include'), target=os.path.join('freetype2')) diff --git a/tools/ports/libjpeg.py b/tools/ports/libjpeg.py index b5ff894cd3597..c948193d57f71 100644 --- a/tools/ports/libjpeg.py +++ b/tools/ports/libjpeg.py @@ -5,7 +5,6 @@ import os import logging -from pathlib import Path VERSION = '9c' HASH = 'b2affe9a1688bd49fc033f4682c4a242d4ee612f1affaef532f5adcb4602efc4433c4a52a4b3d69e7440ff1f6413b1b041b419bc90efd6d697999961a9a6afb7' @@ -24,7 +23,7 @@ def get(ports, settings, shared): def create(final): logging.info('building port: libjpeg') source_path = os.path.join(ports.get_dir(), 'libjpeg', 'jpeg-9c') - Path(source_path, 'jconfig.h').write_text(jconfig_h) + ports.write_file(os.path.join(source_path, 'jconfig.h'), jconfig_h) ports.install_headers(source_path) excludes = [ 'ansi2knr.c', 'cjpeg.c', 'ckconfig.c', 'djpeg.c', 'example.c', diff --git a/tools/ports/libmodplug.py b/tools/ports/libmodplug.py index 687abb81c66ab..8aee0b6700667 100644 --- a/tools/ports/libmodplug.py +++ b/tools/ports/libmodplug.py @@ -5,7 +5,6 @@ import os import logging -from pathlib import Path TAG = '11022021' HASH = 'f770031ad6c2152cbed8c8eab8edf2be1d27f9e74bc255a9930c17019944ee5fdda5308ea992c66a78af9fe1d8dca090f6c956910ce323f8728247c10e44036b' @@ -25,7 +24,7 @@ def create(final): src_dir = os.path.join(source_path, 'src') libmodplug_path = os.path.join(src_dir, 'libmodplug') - Path(source_path, 'config.h').write_text(config_h) + ports.write_file(os.path.join(source_path, 'config.h'), config_h) flags = [ '-Wno-deprecated-register', diff --git a/tools/ports/libpng.py b/tools/ports/libpng.py index 2dbca2aa47543..2ec5ffe9c5852 100644 --- a/tools/ports/libpng.py +++ b/tools/ports/libpng.py @@ -5,7 +5,6 @@ import os import logging -from pathlib import Path TAG = '1.6.37' HASH = '2ce2b855af307ca92a6e053f521f5d262c36eb836b4810cb53c809aa3ea2dcc08f834aee0ffd66137768a54397e28e92804534a74abb6fc9f6f3127f14c9c338' @@ -30,7 +29,7 @@ def create(final): logging.info('building port: libpng') source_path = os.path.join(ports.get_dir(), 'libpng', 'libpng-' + TAG) - Path(source_path, 'pnglibconf.h').write_text(pnglibconf_h) + ports.write_file(os.path.join(source_path, 'pnglibconf.h'), pnglibconf_h) ports.install_headers(source_path) flags = ['-sUSE_ZLIB=1'] diff --git a/tools/ports/mpg123.py b/tools/ports/mpg123.py index b8f221309fadf..dab48d651afa4 100644 --- a/tools/ports/mpg123.py +++ b/tools/ports/mpg123.py @@ -5,7 +5,6 @@ import os import logging -from pathlib import Path TAG = '1.26.2' HASH = 'aa63fcb08b243a1e09f7701b3d84a19d7412a87253d54d49f014fdb9e75bbc81d152a41ed750fccde901453929b2a001585a7645351b41845ad205c17a73dcc9' @@ -27,8 +26,8 @@ def create(final): libmpg123_path = os.path.join(src_path, 'libmpg123') compat_path = os.path.join(src_path, 'compat') - Path(src_path, 'config.h').write_text(config_h) - Path(libmpg123_path, 'mpg123.h').write_text(mpg123_h) + ports.write_file(os.path.join(src_path, 'config.h'), config_h) + ports.write_file(os.path.join(libmpg123_path, 'mpg123.h'), mpg123_h) # copy header to a location so it can be used as 'MPG123/' ports.install_headers(libmpg123_path, pattern="*123.h", target='') diff --git a/tools/ports/ogg.py b/tools/ports/ogg.py index b6e5c817882dd..f4d131d4d5b0b 100644 --- a/tools/ports/ogg.py +++ b/tools/ports/ogg.py @@ -5,7 +5,6 @@ import logging import os -from pathlib import Path TAG = 'version_1' HASH = '929e8d6003c06ae09593021b83323c8f1f54532b67b8ba189f4aedce52c25dc182bac474de5392c46ad5b0dea5a24928e4ede1492d52f4dd5cd58eea9be4dba7' @@ -22,7 +21,7 @@ def create(final): logging.info('building port: ogg') source_path = os.path.join(ports.get_dir(), 'ogg', 'Ogg-' + TAG) - Path(source_path, 'include', 'ogg', 'config_types.h').write_text(config_types_h) + ports.write_file(os.path.join(source_path, 'include', 'ogg', 'config_types.h'), config_types_h) ports.install_header_dir(os.path.join(source_path, 'include', 'ogg'), 'ogg') ports.build_port(os.path.join(source_path, 'src'), final, 'ogg') diff --git a/tools/ports/zlib.py b/tools/ports/zlib.py index 2a4f6fe1efc32..727302201e3c6 100644 --- a/tools/ports/zlib.py +++ b/tools/ports/zlib.py @@ -4,7 +4,6 @@ # found in the LICENSE file. import os -from pathlib import Path VERSION = '1.2.12' HASH = 'cc2366fa45d5dfee1f983c8c51515e0cff959b61471e2e8d24350dea22d3f6fcc50723615a911b046ffc95f51ba337d39ae402131a55e6d1541d3b095d6c0a14' @@ -19,7 +18,7 @@ def get(ports, settings, shared): def create(final): source_path = os.path.join(ports.get_dir(), 'zlib', 'zlib-' + VERSION) - Path(source_path, 'zconf.h').write_text(zconf_h) + ports.write_file(os.path.join(source_path, 'zconf.h'), zconf_h) ports.install_headers(source_path) # build diff --git a/tools/system_libs.py b/tools/system_libs.py index d7ebcb2acc35c..3e8f70b5b5dff 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -28,6 +28,14 @@ 'getsockopt.c', 'setsockopt.c', 'freeaddrinfo.c', 'in6addr_any.c', 'in6addr_loopback.c', 'accept4.c'] +# Experimental: Setting EMCC_USE_NINJA will cause system libraries to get built with ninja rather +# than simple subprocesses. The primary benefit here is that we get accurate dependency tracking. +# This means we can avoid completely rebuilding a library and just rebuild based on what changed. +# +# Setting EMCC_USE_NINJA=2 means that ninja will automatically be run for each library needed at +# link time. +USE_NINJA = int(os.environ.get('EMCC_USE_NINJA', '0')) + def files_in_path(path, filenames): srcdir = utils.path_from_root(path) @@ -93,6 +101,115 @@ def create_lib(libname, inputs): building.emar('cr', libname, inputs) +def run_ninja(build_dir): + diagnostics.warning('experimental', 'ninja support is experimental') + cmd = ['ninja', '-C', build_dir] + if shared.PRINT_STAGES: + cmd.append('-v') + shared.check_call(cmd, env=clean_env()) + + +def create_ninja_file(input_files, filename, libname, cflags, asflags=None, customize_build_flags=None): + if asflags is None: + asflags = [] + # TODO(sbc) There is an llvm bug that causes a crash when `-g` is used with + # assembly files that define wasm globals. + asflags = [arg for arg in asflags if arg != '-g'] + cflags_asm = [arg for arg in cflags if arg != '-g'] + + def join(flags): + return ' '.join(flags) + + out = f'''\ +# Automatically generated by tools/system_libs.py. DO NOT EDIT + +ninja_required_version = 1.5 + +ASFLAGS = {join(asflags)} +CFLAGS = {join(cflags)} +CFLAGS_ASM = {join(cflags_asm)} +EMCC = {shared.EMCC} +EMXX = {shared.EMXX} +EMAR = {shared.EMAR} + +rule cc + depfile = $out.d + command = $EMCC -MD -MF $out.d $CFLAGS -c $in -o $out + description = CC $out + +rule cxx + depfile = $out.d + command = $EMXX -MD -MF $out.d $CFLAGS -c $in -o $out + description = CXX $out + +rule asm + command = $EMCC $ASFLAGS -c $in -o $out + description = ASM $out + +rule asm_cpp + depfile = $out.d + command = $EMCC -MD -MF $out.d $CFLAGS_ASM -c $in -o $out + description = ASM $out + +rule direct_cc + depfile = $with_depfile + command = $EMCC -MD -MF $with_depfile $CFLAGS -c $in -o $out + description = CC $out + +rule archive + command = $EMAR cr $out $in + description = AR $out + +''' + suffix = shared.suffix(libname) + + case_insensitive = is_case_insensitive(os.path.dirname(filename)) + if suffix == '.o': + assert len(input_files) == 1 + depfile = shared.unsuffixed_basename(input_files[0]) + '.d' + out += f'build {libname}: direct_cc {input_files[0]}\n' + out += f' with_depfile = {depfile}\n' + else: + objects = [] + for src in input_files: + # Resolve duplicates by appending unique. + # This is needed on case insensitve filesystem to handle, + # for example, _exit.o and _Exit.o. + o = shared.unsuffixed_basename(src) + '.o' + object_uuid = 0 + if case_insensitive: + o = o.lower() + # Find a unique basename + while o in objects: + object_uuid += 1 + o = f'{o}__{object_uuid}.o' + objects.append(o) + ext = shared.suffix(src) + if ext == '.s': + out += f'build {o}: asm {src}\n' + flags = asflags + elif ext == '.S': + out += f'build {o}: asm_cpp {src}\n' + flags = cflags_asm + elif ext == '.c': + out += f'build {o}: cc {src}\n' + flags = cflags + else: + out += f'build {o}: cxx {src}\n' + flags = cflags + if customize_build_flags: + custom_flags = customize_build_flags(flags, src) + if custom_flags != flags: + out += f' CFLAGS = {join(custom_flags)}' + out += '\n' + + objects = sorted(objects, key=lambda x: os.path.basename(x)) + objects = ' '.join(objects) + out += f'build {libname}: archive {objects}\n' + + utils.write_file(filename, out) + + def is_case_insensitive(path): """Returns True if the filesystem at `path` is case insensitive.""" utils.write_file(os.path.join(path, 'test_file'), '') @@ -240,15 +357,18 @@ def can_build(self): return True def erase(self): - shared.Cache.erase_file(shared.Cache.get_lib_name(self.get_filename())) + shared.Cache.erase_file(self.get_path()) - def get_path(self): + def get_path(self, absolute=False): + return shared.Cache.get_lib_name(self.get_filename(), absolute=absolute) + + def build(self): """ Gets the cached path of this library. This will trigger a build if this library is not in the cache. """ - return shared.Cache.get_lib(self.get_filename(), self.build) + return shared.Cache.get(self.get_path(), self.do_build, force=USE_NINJA == 2, quiet=USE_NINJA) def get_link_flag(self): """ @@ -256,7 +376,7 @@ def get_link_flag(self): This will trigger a build if this library is not in the cache. """ - fullpath = self.get_path() + fullpath = self.build() # For non-libaries (e.g. crt1.o) we pass the entire path to the linker if self.get_ext() != '.a': return fullpath @@ -282,6 +402,17 @@ def get_files(self): raise NotImplementedError() + def build_with_ninja(self, build_dir, libname): + ensure_sysroot() + utils.safe_ensure_dirs(build_dir) + + cflags = self.get_cflags() + asflags = get_base_cflags() + input_files = self.get_files() + ninja_file = os.path.join(build_dir, 'build.ninja') + create_ninja_file(input_files, ninja_file, libname, cflags, asflags=asflags, customize_build_flags=self.customize_build_cmd) + run_ninja(build_dir) + def build_objects(self, build_dir): """ Returns a list of compiled object files for this library. @@ -335,13 +466,20 @@ def customize_build_cmd(self, cmd, filename): # noqa For example, libc uses this to replace -Oz with -O2 for some subset of files.""" return cmd - def build(self, out_filename): + def do_build(self, out_filename): """Builds the library and returns the path to the file.""" - build_dir = shared.Cache.get_path(os.path.join('build', self.get_base_name())) - utils.safe_ensure_dirs(build_dir) - create_lib(out_filename, self.build_objects(build_dir)) - if not shared.DEBUG: - utils.delete_dir(build_dir) + assert out_filename == self.get_path(absolute=True) + build_dir = os.path.join(shared.Cache.get_path('build'), self.get_base_name()) + if USE_NINJA: + self.build_with_ninja(build_dir, out_filename) + else: + # Use a seperate build directory to the ninja flavor so that building without + # EMCC_USE_NINJA doesn't clobber the ninja build tree + build_dir += '-tmp' + utils.safe_ensure_dirs(build_dir) + create_lib(out_filename, self.build_objects(build_dir)) + if not shared.DEBUG: + utils.delete_dir(build_dir) @classmethod def _inherit_list(cls, attr):