From 547d991f4f58c8a0f8a410b6f4c572113103cc0e Mon Sep 17 00:00:00 2001 From: Wataru Ishida Date: Mon, 28 Nov 2022 01:02:24 +0900 Subject: [PATCH] builds: support rootfs.sqsh over 2GB ONL's switool.py and swiprep don't support files over 2GB. Add Goldstone version to support it. Signed-off-by: Wataru Ishida --- builds/amd64/swi/builds/Makefile | 2 +- builds/arm64/swi/builds/Makefile | 2 +- make/swi.mk | 39 ++++++- .../python/goldstone/install/BaseInstall.py | 36 +++++- tools/switool.py | 104 ++++++++++++++++++ 5 files changed, 175 insertions(+), 8 deletions(-) create mode 100755 tools/switool.py diff --git a/builds/amd64/swi/builds/Makefile b/builds/amd64/swi/builds/Makefile index 044eea6..6feb54a 100644 --- a/builds/amd64/swi/builds/Makefile +++ b/builds/amd64/swi/builds/Makefile @@ -1,3 +1,3 @@ ROOTFS_PACKAGE := goldstone-rootfs include $(X1)/make/config.amd64.mk -include $(ONL)/make/swi.mk +include $(X1)/make/swi.mk diff --git a/builds/arm64/swi/builds/Makefile b/builds/arm64/swi/builds/Makefile index 37d174a..686a623 100644 --- a/builds/arm64/swi/builds/Makefile +++ b/builds/arm64/swi/builds/Makefile @@ -1,3 +1,3 @@ ROOTFS_PACKAGE := goldstone-rootfs include $(X1)/make/config.arm64.mk -include $(ONL)/make/swi.mk +include $(X1)/make/swi.mk diff --git a/make/swi.mk b/make/swi.mk index 5bd3ad8..2ab9bb5 100644 --- a/make/swi.mk +++ b/make/swi.mk @@ -1,3 +1,36 @@ -ONL_PRE_SWITOOL_CMDS := $(X1)/tools/zerotouch.py --manifest-version 1 --operation swi --manifest manifest.json > zerotouch.json -ONL_SWITOOL_EXTRA_ARGS := --add-files zerotouch.json -include $(ONL)/make/swi.mk +ifndef ARCH +$(error $$ARCH is not set) +endif + +ifndef ROOTFS_PACKAGE +$(error $$ROOTFS_PACKAGE not set.) +endif + +ifdef REBUILD_RFS +FORCE_OPTION = --force +endif + +ifndef SWI_WORKDIR +SWI_WORKDIR := $(ONL_DEBIAN_SUITE) +endif + +LINK_OPTIONS := $(FORCE_OPTION) --link-file $(ROOTFS_PACKAGE):$(ARCH) rootfs-$(ARCH).sqsh . --link-file $(ROOTFS_PACKAGE):$(ARCH) manifest.json . + +ifndef FILENAMER +FILENAMER := $(ONL)/tools/filenamer.py +endif + +swi: FORCE clean + mkdir $(SWI_WORKDIR) && cd $(SWI_WORKDIR) && $(ONLPM) $(LINK_OPTIONS) +ifdef ONL_PRE_SWITOOL_CMDS + $(ONL_V_at) $(ONL_PRE_SWITOOL_CMDS) +endif + $(ONL_V_at) cd $(SWI_WORKDIR) && $(X1)/tools/switool.py --create $(ONL_SWITOOL_EXTRA_ARGS) --rootfs rootfs-$(ARCH).sqsh --manifest manifest.json $@ + $(ONL_V_at) cd $(SWI_WORKDIR) && mv $@ `$(FILENAMER) --type swi --manifest manifest.json $@` + $(ONL_V_at) cd $(SWI_WORKDIR) && for f in `ls *.swi`; do md5sum $$f > $$f.md5sum; done + $(ONL_V_at) rm -rf $(SWI_WORKDIR)/rootfs-$(ARCH).sqsh + $(ONL_V_at) ls $(SWI_WORKDIR)/*.swi +FORCE: + +clean: + rm -rf $(SWI_WORKDIR) diff --git a/packages/base/all/initrd/loader-files/src/python/goldstone/install/BaseInstall.py b/packages/base/all/initrd/loader-files/src/python/goldstone/install/BaseInstall.py index bad2034..6d49f66 100644 --- a/packages/base/all/initrd/loader-files/src/python/goldstone/install/BaseInstall.py +++ b/packages/base/all/initrd/loader-files/src/python/goldstone/install/BaseInstall.py @@ -1,8 +1,37 @@ import os +import zipfile +import glob from onl.install import BaseInstall as ONLBaseInstall from onl.install.InstallUtils import MountContext +# The ONL swiprep script uses GNU unzip command, which doesn't support zip64 +def swiprep(self, swi, dst): + z = zipfile.ZipFile(swi) + rootfs = None + for n in z.namelist(): + if n.startswith("rootfs-") and n.endswith(".sqsh"): + rootfs = n + break + if not rootfs: + self.log.error("cannot find a valid rootfs") + return + # the pwd is on tmpfs. copy the rootfs.sqsh to the disk so that it doesn't consume memory + z.extract(rootfs, dst) + self.check_call(("unsquashfs", "-f", "-d", dst, "{}/{}".format(dst, rootfs))) + self.unlink("{}/{}".format(dst, rootfs)) + + self.check_call(("mkdir", "-p", "{}/etc/onl".format(dst))) + + for thing in glob.glob("/etc/onl/*"): + if thing == "/etc/onl/sysconfig": + continue + self.check_call(("cp", "-R", thing, "{}/etc/onl".format(dst))) + + if os.path.exists("/etc/fw_env.config"): + self.copyfile("/etc/fw_env.config", "{}/etc/fw_env.config".format(dst)) + + # Unlike the ONL default behavior, which just installs the SWI blob to the # ONL-IMAGE partition and let the ONL initrd loader extract it to the ONL-DATA # partition, installSwi() extract the SWI now without copying it to the ONL-IMAGE @@ -38,11 +67,12 @@ def installSwi(self): with open("/etc/onl/platform", "w") as f: f.write(onie_platform.replace("_", "-")) - self.installerCopy(base, "swi") - data = self.blkidParts["ONL-DATA"] with MountContext(data.device, log=self.log) as ctx: - self.check_call(("swiprep", "--install", "./swi", ctx.dir)) + # the pwd is on tmpfs. copy the swi to the disk so that it doesn't consume memory + self.installerCopy(base, "{}/swi".format(ctx.dir)) + swiprep(self, "{}/swi".format(ctx.dir), ctx.dir) + self.unlink("{}/swi".format(ctx.dir)) class GrubInstaller(ONLBaseInstall.GrubInstaller, object): diff --git a/tools/switool.py b/tools/switool.py new file mode 100755 index 0000000..f666f7e --- /dev/null +++ b/tools/switool.py @@ -0,0 +1,104 @@ +#!/usr/bin/python2 +############################################################ +import argparse +import sys +import os +import zipfile +import json +import apt_inst +import onlu +import subprocess + +logger = onlu.init_logging("switool") + + +class OnlSwitchImage(object): + def __init__(self, fname, mode): + self.fname = fname + self.mode = mode + # use allowZip64 to support rootfs over 2GB + self.zipfile = zipfile.ZipFile(fname, mode=mode, allowZip64=True) + self.manifest = None + + def add(self, fname, arcname=None, compressed=True): + self.zipfile.write( + fname, + arcname=arcname, + compress_type=zipfile.ZIP_DEFLATED if compressed else zipefile.ZIP_STORED, + ) + + def add_rootfs(self, rootfs_sqsh): + self.add(rootfs_sqsh) + + def add_manifest(self, manifest): + self.add(manifest, arcname="manifest.json") + + def get_manifest(self): + if "manifest.json" in self.zipfile.namelist(): + return json.load(self.zipfile.open("manifest.json")) + else: + return None + + def get_arch(self): + return self.get_manifest()["arch"] + + def get_platforms(self): + p = self.get_manifest()["platforms"] + if type(p) is list: + return p + else: + return p.split(",") + + def get_contents(self): + return self.zipfile.namelist() + + +############################################################ + +ap = argparse.ArgumentParser(description="SWI Tool") +ap.add_argument("--create", action="store_true", help="Create new SWI.") +ap.add_argument("--overwrite", action="store_true", help="Overwrite existing file.") +ap.add_argument("--rootfs", help="Root SquashFS File") +ap.add_argument("--manifest", help="SWI Manifest file.") +ap.add_argument("--add-files", help="Add additional files.", default=[], nargs="+") +ap.add_argument("--contents", help="Show SWI contents.", action="store_true") +ap.add_argument("--platforms", help="Show SWI contents.", action="store_true") +ap.add_argument("swi", help="SWI image name.") + +ops = ap.parse_args() + +if os.path.exists(ops.swi): + if ops.create and not ops.overwrite: + logger.critical("File '%s' exists." % ops.swi) + sys.exit(1) + +swi = None + +if ops.create or ops.overwrite: + if not ops.rootfs: + logger.critical("Rootfs required to create new SWI.") + sys.exit(1) + + if not ops.manifest: + logger.critical("Manifest required to create new SWI.") + sys.exit(1) + + swi = OnlSwitchImage(ops.swi, "w") + swi.add_rootfs(ops.rootfs) + swi.add_manifest(ops.manifest) + for f in ops.add_files: + swi.add(f, arcname=f) + +if swi is None: + + if not os.path.exists(ops.swi): + logger.critical("SWI file %s does not exist." % ops.swi) + sys.exit(1) + + swi = OnlSwitchImage(ops.swi, "r") + +if ops.contents: + print(" ".join(swi.get_contents())) + +if ops.platforms: + print(" ".join(swi.get_platforms()))