Libzim wasm compilation

.github/scripts/build_projects.py

@@ -6,9 +6,11 @@
     make_archive,
     create_desktop_image,
     fix_macos_rpath,
+    upload_archive,
     OS_NAME,
     PLATFORM_TARGET,
     DESKTOP,
+    DEV_BRANCH,
 )
 
 if (PLATFORM_TARGET.startswith("android_")
@@ -32,14 +34,18 @@
     TARGETS = ("kiwix-tools",)
 elif PLATFORM_TARGET == "flatpak":
     TARGETS = ("kiwix-desktop",)
+elif PLATFORM_TARGET == "wasm":
+    TARGETS = ("libzim", )
 else:
     TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
 
 for target in TARGETS:
     run_kiwix_build(target, platform=PLATFORM_TARGET)
     if target == "kiwix-desktop":
-        create_desktop_image(make_release=False)
+        archive = create_desktop_image(make_release=False)
     else:
         if PLATFORM_TARGET == "native_mixed" and OS_NAME == "osx":
             fix_macos_rpath(target)
-        make_archive(target, make_release=False)
+        archive = make_archive(target, make_release=False)
+    if archive:
+        upload_archive(archive, target, make_release=False, dev_branch=DEV_BRANCH)

.github/scripts/build_release_nightly.py

@@ -38,6 +38,8 @@
     TARGETS = ("kiwix-tools",)
 elif PLATFORM_TARGET == "flatpak":
     TARGETS = ("kiwix-desktop",)
+elif PLATFORM_TARGET == "wasm":
+    TARGETS = ("libzim", )
 else:
     TARGETS = ("libzim", "zim-tools", "libkiwix", "kiwix-tools")
 

.github/scripts/common.py

@@ -39,6 +39,11 @@
 MAKE_RELEASE = re.fullmatch(r"r_[0-9]+", _ref) is not None
 MAKE_RELEASE = MAKE_RELEASE and (_environ.get('GITHUB_EVENT_NAME') != 'schedule')
 
+if not MAKE_RELEASE and _ref != "master":
+    DEV_BRANCH = _ref
+else:
+    DEV_BRANCH = None
+
 RELEASE_OS_NAME = "macos" if OS_NAME == "osx" else "linux"
 
 PLATFORM_TO_RELEASE = {
@@ -51,13 +56,7 @@
     "android_arm64": "android-arm64",
     "android_x86": "android-x86",
     "android_x86_64": "android-x86_64",
-}
-
-LIB_PREFIX = {
-    "android_arm": "arm-linux-androideabi",
-    "android_arm64": "aarch64-linux-android",
-    "android_x86": "i686-linux-android",
-    "android_x86_64": "x86_64-linux-android",
+    "wasm": "wasm-emscripten",
 }
 
 FLATPAK_HTTP_GIT_REMOTE = "https://github.com/flathub/org.kiwix.desktop.git"
@@ -94,15 +93,12 @@
     "libzim": (
         INSTALL_DIR,
         (
-            "lib/{libprefix}/libzim.so".format(
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
-            ),
-            "lib/{libprefix}/libzim.so.{version}".format(
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
+            "lib/*/libzim.a",
+            "lib/*/libzim.so",
+            "lib/*/libzim.so.{version}".format(
                 version=main_project_versions["libzim"]
             ),
-            "lib/{libprefix}/libzim.so.{version}".format(
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
+            "lib/*/libzim.so.{version}".format(
                 version=main_project_versions["libzim"][0]
             ),
             "lib/libzim.{}.dylib".format(
@@ -115,15 +111,11 @@
     "libkiwix": (
         INSTALL_DIR,
         (
-            "lib/{libprefix}/libkiwix.so".format(
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
-            ),
-            "lib/{libprefix}/libkiwix.so.{version}".format(
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
+            "lib/*/libkiwix.so",
+            "lib/*/libkiwix.so.{version}".format(
                 version=main_project_versions["libkiwix"]
             ),
-            "lib/{libprefix}/libkiwix.so.{version}".format(
-                libprefix=LIB_PREFIX.get(PLATFORM_TARGET, "x86_64-linux-gnu"),
+            "lib/*/libkiwix.so.{version}".format(
                 version=main_project_versions["libkiwix"][0]
             ),
             "include/kiwix/**/*.h"
@@ -240,7 +232,7 @@ def upload(file_to_upload, host, dest_path):
     subprocess.check_call(command)
 
 
-def upload_archive(archive, project, make_release):
+def upload_archive(archive, project, make_release, dev_branch=None):
     if not archive.exists():
         print_message("No archive {} to upload!", archive)
         return
@@ -257,9 +249,12 @@ def upload_archive(archive, project, make_release):
     else:
         dest_path = dest_path + "nightly/" + DATE
 
-    # Make the archive read only. This way, scp will preserve rights.
-    # If somehow we try to upload twice the same archive, scp will fails.
-    archive.chmod(0o444)
+    if dev_branch:
+        dest_path = "/data/tmp/ci/" + dev_branch
+    else:
+        # Make the archive read only. This way, scp will preserve rights.
+        # If somehow we try to upload twice the same archive, scp will fails.
+        archive.chmod(0o444)
 
     upload(archive, host, dest_path)
 
@@ -280,6 +275,7 @@ def make_deps_archive(target=None, name=None, full=False):
         if (base_dir / "meson_cross_file.txt").exists():
             files_to_archive.append(base_dir / "meson_cross_file.txt")
     files_to_archive += HOME.glob("BUILD_*/android-ndk*")
+    files_to_archive += HOME.glob("BUILD_*/emsdk*")
     if (BASE_DIR / "meson_cross_file.txt").exists():
         files_to_archive.append(BASE_DIR / "meson_cross_file.txt")
 

.github/scripts/compile_all_deps.py

@@ -14,6 +14,8 @@
 
 if PLATFORM_TARGET.startswith("android_") or PLATFORM_TARGET.startswith("iOS"):
     TARGETS = ("libzim", "libkiwix")
+elif PLATFORM_TARGET == "wasm":
+    TARGETS = ("libzim", )
 elif PLATFORM_TARGET.startswith("native_"):
     if OS_NAME == "osx":
         TARGETS = ("libzim", "zim-tools", "libkiwix")

.github/workflows/ci.yml

@@ -38,6 +38,7 @@ jobs:
           - native_dyn
           - native_mixed
           - native_desktop
+          - wasm
           - armhf_static
           - armhf_dyn
           - i586_static

kiwixbuild/buildenv.py

@@ -148,3 +148,21 @@ def get_env(self, *, cross_comp_flags, cross_compilers, cross_path):
         if cross_path:
             env['PATH'] = ':'.join(self.platformInfo.get_bin_dir() + [env['PATH']])
         return env
+
+
+    @property
+    def configure_wrapper(self):
+        wrapper = getattr(self.platformInfo, "configure_wrapper", "")
+        if wrapper:
+            return "{} ".format(wrapper)
+        else:
+            return ""
+
+    @property
+    def make_wrapper(self):
+        wrapper = getattr(self.platformInfo, "make_wrapper", "")
+        if wrapper:
+            return "{} ".format(wrapper)
+        else:
+            return ""
+

kiwixbuild/dependencies/__init__.py

@@ -6,6 +6,7 @@
     aria2,
     armhf,
     docoptcpp,
+    emsdk,
     flatpak,
     gumbo,
     icu4c,

kiwixbuild/dependencies/all_dependencies.py

@@ -12,6 +12,9 @@ class AllBaseDependencies(Dependency):
     class Builder(NoopBuilder):
         @classmethod
         def get_dependencies(cls, platformInfo, allDeps):
+            if platformInfo.build == "wasm":
+                return ['zlib', 'lzma', 'zstd', 'icu4c', 'xapian-core']
+
             base_deps = ['zlib', 'lzma', 'zstd', 'xapian-core', 'pugixml', 'libcurl', 'icu4c', 'mustache', 'libmicrohttpd', 'zim-testing-suite']
             # Add specific dependencies depending of the platform
             if platformInfo.build not in ('android', 'iOS'):

kiwixbuild/dependencies/base.py

@@ -70,8 +70,9 @@ def _log_dir(self):
     def _patch(self, context):
         context.try_skip(self.source_path)
         for p in self.patches:
-            with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
-                run_command("patch -p1", self.source_path, context, input=patch_input.read())
+            patch_file_path = pj(SCRIPT_DIR, 'patches', p)
+            patch_command = "patch -p1 -i {patch}".format(patch=patch_file_path)
+            run_command(patch_command, self.source_path, context)
 
     def command(self, name, function, *args):
         print("  {} {} : ".format(name, self.name), end="", flush=True)
@@ -339,7 +340,7 @@ class MakeBuilder(Builder):
 
     @property
     def make_install_target(self):
-        if self.buildEnv.platformInfo.build == 'iOS':
+        if self.buildEnv.platformInfo.build in ('iOS', "wasm"):
             return 'install'
         return 'install-strip'
 
@@ -368,8 +369,9 @@ def set_configure_env(self, env):
 
     def _configure(self, context):
         context.try_skip(self.build_path)
-        command = "{configure_script} {configure_option}"
+        command = "{configure_wrapper}{configure_script} {configure_option}"
         command = command.format(
+            configure_wrapper=self.buildEnv.configure_wrapper,
             configure_script=pj(self.source_path, self.configure_script),
             configure_option=self.all_configure_option
         )
@@ -379,7 +381,8 @@ def _configure(self, context):
 
     def _compile(self, context):
         context.try_skip(self.build_path)
-        command = "make -j4 {make_target} {make_option}".format(
+        command = "{make_wrapper}make -j4 {make_target} {make_option}".format(
+            make_wrapper=self.buildEnv.make_wrapper,
             make_target=self.make_target,
             make_option=self.make_option
         )
@@ -388,7 +391,8 @@ def _compile(self, context):
 
     def _install(self, context):
         context.try_skip(self.build_path)
-        command = "make {make_install_target} {make_option}".format(
+        command = "{make_wrapper}make {make_install_target} {make_option}".format(
+            make_wrapper=self.buildEnv.make_wrapper,
             make_install_target=self.make_install_target,
             make_option=self.make_option
         )
@@ -397,7 +401,9 @@ def _install(self, context):
 
     def _make_dist(self, context):
         context.try_skip(self.build_path)
-        command = "make dist"
+        command = "{make_wrapper}make dist".format(
+            make_wrapper=self.buildEnv.make_wrapper
+        )
         env = self.get_env(cross_comp_flags=True, cross_compilers=True, cross_path=True)
         run_command(command, self.build_path, context, env=env)
 

kiwixbuild/dependencies/emsdk.py

@@ -0,0 +1,47 @@
+import os
+
+from .base import Dependency, ReleaseDownload, Builder
+from kiwixbuild.utils import Remotefile, run_command, copy_tree
+
+pj = os.path.join
+
+class emsdk(Dependency):
+    dont_skip = True
+    neutral = False
+    name = 'emsdk'
+
+    class Source(ReleaseDownload):
+        archive = Remotefile('emsdk-3.1.24.tar.gz',
+                             '1aa5365ccb2147701cc9d1e59a5a49577c1d6aea55da7c450df2d5ffa48b8a58',
+                             'https://codeload.github.com/emscripten-core/emsdk/tar.gz/refs/tags/3.1.24')
+
+        @property
+        def source_dir(self):
+            return self.target.full_name()
+
+
+    class Builder(Builder):
+        @property
+        def install_path(self):
+            return self.build_path
+
+        def _copy_source(self, context):
+            context.try_skip(self.build_path)
+            copy_tree(self.source_path, self.build_path)
+
+        def _install(self, context):
+            context.try_skip(self.build_path)
+            command = "./emsdk install 3.1.24"
+            run_command(command, self.build_path, context)
+
+        def _activate(self, context):
+            context.try_skip(self.build_path)
+            command = "./emsdk activate 3.1.24"
+            run_command(command, self.build_path, context)
+
+
+        def build(self):
+            self.command('copy_source', self._copy_source)
+            self.command('install', self._install)
+            self.command('activate', self._activate)
+

kiwixbuild/dependencies/icu4c.py

@@ -18,7 +18,9 @@ class Source(ReleaseDownload):
                    "icu4c_custom_data.patch",
                    "icu4c_noxlocale.patch",
                    "icu4c_rpath.patch",
-                   "icu4c_build_config.patch"]
+                   "icu4c_build_config.patch",
+                   "icu4c_wasm.patch"
+                  ]
 
 
     class Builder(MakeBuilder):
@@ -42,6 +44,6 @@ def configure_option(self):
                     'native_static' if platformInfo.static else 'native_dyn')
                 options += " --with-cross-build={} --disable-tools".format(
                     icu_native_builder.build_path)
-            if platformInfo.build == 'android':
+            if platformInfo.build in ('android', 'wasm'):
                 options += " --with-data-packaging=archive"
             return options

kiwixbuild/dependencies/libzim.py

@@ -19,7 +19,7 @@ class Builder(MesonBuilder):
         @classmethod
         def get_dependencies(cls, platformInfo, allDeps):
             deps = ['lzma', 'zstd', 'xapian-core', 'icu4c']
-            if platformInfo.build != 'flatpak':
+            if platformInfo.name not in ('flatpak', 'wasm'):
                 deps.append('zim-testing-suite')
             return deps
 
@@ -37,7 +37,9 @@ def configure_option(self):
             if platformInfo.name == "flatpak":
                 config_options.append("--wrap-mode=nodownload")
                 config_options.append("-Dtest_data_dir=none")
-            else:
+            if platformInfo.name == "wasm":
+                config_options.append("-Dexamples=false")
+            if platformInfo.name not in ("flatpak", "wasm"):
                 zim_testing_suite = get_target_step('zim-testing-suite', 'source')
                 config_options.append('-Dtest_data_dir={}'.format(zim_testing_suite.source_path))
             return " ".join(config_options)

kiwixbuild/dependencies/lzma.py

@@ -9,11 +9,20 @@ class lzma(Dependency):
     name = 'lzma'
 
     class Source(ReleaseDownload):
-        archive = Remotefile('xz-5.2.4.tar.gz',
-                             'b512f3b726d3b37b6dc4c8570e137b9311e7552e8ccbab4d39d47ce5f4177145'
+        archive = Remotefile('xz-5.2.6.tar.gz',
+                             'a2105abee17bcd2ebd15ced31b4f5eda6e17efd6b10f921a01cda4a44c91b3a0',
+                             'https://altushost-swe.dl.sourceforge.net/project/lzmautils/xz-5.2.6.tar.gz'
                             )
 
     class Builder(MakeBuilder):
         @property
         def configure_option(self):
-            return "--disable-xz --disable-xzdec"
+            return ("--disable-xz "
+                    "--disable-xzdec "
+                    "--disable-lzmadec "
+                    "--disable-lzmainfo "
+                    "--disable-lzma-links "
+                    "--disable-scripts "
+                    "--disable-doc "
+#                    "--disable-symbol-versions "
+                  )

kiwixbuild/dependencies/xapian.py

@@ -24,7 +24,7 @@ class Builder(MakeBuilder):
         @classmethod
         def get_dependencies(cls, platformInfo, allDeps):
             deps = ['zlib', 'lzma']
-            if (platformInfo.build == 'win32'
+            if (platformInfo.build in ('win32', 'wasm')
              or neutralEnv('distname') == 'Darwin'):
                 return deps
             return deps + ['uuid']