builds: support rootfs.sqsh over 2GB

ONL's switool.py and swiprep don't support files over 2GB.
Add Goldstone version to support it.

Signed-off-by: Wataru Ishida <[email protected]>

builds/amd64/swi/builds/Makefile

@@ -1,3 +1,3 @@
 ROOTFS_PACKAGE := goldstone-rootfs
 include $(X1)/make/config.amd64.mk
-include $(ONL)/make/swi.mk
+include $(X1)/make/swi.mk

builds/arm64/swi/builds/Makefile

@@ -1,3 +1,3 @@
 ROOTFS_PACKAGE := goldstone-rootfs
 include $(X1)/make/config.arm64.mk
-include $(ONL)/make/swi.mk
+include $(X1)/make/swi.mk

make/swi.mk

@@ -1,3 +1,36 @@
-ONL_PRE_SWITOOL_CMDS := $(X1)/tools/zerotouch.py --manifest-version 1 --operation swi --manifest manifest.json > zerotouch.json
-ONL_SWITOOL_EXTRA_ARGS := --add-files zerotouch.json
-include $(ONL)/make/swi.mk
+ifndef ARCH
+$(error $$ARCH is not set)
+endif
+
+ifndef ROOTFS_PACKAGE
+$(error $$ROOTFS_PACKAGE not set.)
+endif
+
+ifdef REBUILD_RFS
+FORCE_OPTION = --force
+endif
+
+ifndef SWI_WORKDIR
+SWI_WORKDIR := $(ONL_DEBIAN_SUITE)
+endif
+
+LINK_OPTIONS := $(FORCE_OPTION) --link-file $(ROOTFS_PACKAGE):$(ARCH) rootfs-$(ARCH).sqsh . --link-file $(ROOTFS_PACKAGE):$(ARCH) manifest.json .
+
+ifndef FILENAMER
+FILENAMER := $(ONL)/tools/filenamer.py
+endif
+
+swi: FORCE clean
+	mkdir $(SWI_WORKDIR) && cd $(SWI_WORKDIR) && $(ONLPM) $(LINK_OPTIONS)
+ifdef ONL_PRE_SWITOOL_CMDS
+	$(ONL_V_at) $(ONL_PRE_SWITOOL_CMDS)
+endif
+	$(ONL_V_at) cd $(SWI_WORKDIR) && $(X1)/tools/switool.py --create $(ONL_SWITOOL_EXTRA_ARGS) --rootfs rootfs-$(ARCH).sqsh --manifest manifest.json $@
+	$(ONL_V_at) cd $(SWI_WORKDIR) && mv $@ `$(FILENAMER) --type swi --manifest manifest.json $@`
+	$(ONL_V_at) cd $(SWI_WORKDIR) && for f in `ls *.swi`; do md5sum $$f > $$f.md5sum; done
+	$(ONL_V_at) rm -rf $(SWI_WORKDIR)/rootfs-$(ARCH).sqsh
+	$(ONL_V_at) ls $(SWI_WORKDIR)/*.swi
+FORCE:
+
+clean:
+	rm -rf $(SWI_WORKDIR)

packages/base/all/initrd/loader-files/src/python/goldstone/install/BaseInstall.py

@@ -1,8 +1,37 @@
 import os
+import zipfile
+import glob
 
 from onl.install import BaseInstall as ONLBaseInstall
 from onl.install.InstallUtils import MountContext
 
+# The ONL swiprep script uses GNU unzip command, which doesn't support zip64
+def swiprep(self, swi, dst):
+    z = zipfile.ZipFile(swi)
+    rootfs = None
+    for n in z.namelist():
+        if n.startswith("rootfs-") and n.endswith(".sqsh"):
+            rootfs = n
+            break
+    if not rootfs:
+        self.log.error("cannot find a valid rootfs")
+        return
+    # the pwd is on tmpfs. copy the rootfs.sqsh to the disk so that it doesn't consume memory
+    z.extract(rootfs, dst)
+    self.check_call(("unsquashfs", "-f", "-d", dst, "{}/{}".format(dst, rootfs)))
+    self.unlink("{}/{}".format(dst, rootfs))
+
+    self.check_call(("mkdir", "-p", "{}/etc/onl".format(dst)))
+
+    for thing in glob.glob("/etc/onl/*"):
+        if thing == "/etc/onl/sysconfig":
+            continue
+        self.check_call(("cp", "-R", thing, "{}/etc/onl".format(dst)))
+
+    if os.path.exists("/etc/fw_env.config"):
+        self.copyfile("/etc/fw_env.config", "{}/etc/fw_env.config".format(dst))
+
+
 # Unlike the ONL default behavior, which just installs the SWI blob to the
 # ONL-IMAGE partition and let the ONL initrd loader extract it to the ONL-DATA
 # partition, installSwi() extract the SWI now without copying it to the ONL-IMAGE
@@ -38,11 +67,12 @@ def installSwi(self):
     with open("/etc/onl/platform", "w") as f:
         f.write(onie_platform.replace("_", "-"))
 
-    self.installerCopy(base, "swi")
-
     data = self.blkidParts["ONL-DATA"]
     with MountContext(data.device, log=self.log) as ctx:
-        self.check_call(("swiprep", "--install", "./swi", ctx.dir))
+        # the pwd is on tmpfs. copy the swi to the disk so that it doesn't consume memory
+        self.installerCopy(base, "{}/swi".format(ctx.dir))
+        swiprep(self, "{}/swi".format(ctx.dir), ctx.dir)
+        self.unlink("{}/swi".format(ctx.dir))
 
 
 class GrubInstaller(ONLBaseInstall.GrubInstaller, object):

tools/switool.py

@@ -0,0 +1,104 @@
+#!/usr/bin/python2
+############################################################
+import argparse
+import sys
+import os
+import zipfile
+import json
+import apt_inst
+import onlu
+import subprocess
+
+logger = onlu.init_logging("switool")
+
+
+class OnlSwitchImage(object):
+    def __init__(self, fname, mode):
+        self.fname = fname
+        self.mode = mode
+        # use allowZip64 to support rootfs over 2GB
+        self.zipfile = zipfile.ZipFile(fname, mode=mode, allowZip64=True)
+        self.manifest = None
+
+    def add(self, fname, arcname=None, compressed=True):
+        self.zipfile.write(
+            fname,
+            arcname=arcname,
+            compress_type=zipfile.ZIP_DEFLATED if compressed else zipefile.ZIP_STORED,
+        )
+
+    def add_rootfs(self, rootfs_sqsh):
+        self.add(rootfs_sqsh)
+
+    def add_manifest(self, manifest):
+        self.add(manifest, arcname="manifest.json")
+
+    def get_manifest(self):
+        if "manifest.json" in self.zipfile.namelist():
+            return json.load(self.zipfile.open("manifest.json"))
+        else:
+            return None
+
+    def get_arch(self):
+        return self.get_manifest()["arch"]
+
+    def get_platforms(self):
+        p = self.get_manifest()["platforms"]
+        if type(p) is list:
+            return p
+        else:
+            return p.split(",")
+
+    def get_contents(self):
+        return self.zipfile.namelist()
+
+
+############################################################
+
+ap = argparse.ArgumentParser(description="SWI Tool")
+ap.add_argument("--create", action="store_true", help="Create new SWI.")
+ap.add_argument("--overwrite", action="store_true", help="Overwrite existing file.")
+ap.add_argument("--rootfs", help="Root SquashFS File")
+ap.add_argument("--manifest", help="SWI Manifest file.")
+ap.add_argument("--add-files", help="Add additional files.", default=[], nargs="+")
+ap.add_argument("--contents", help="Show SWI contents.", action="store_true")
+ap.add_argument("--platforms", help="Show SWI contents.", action="store_true")
+ap.add_argument("swi", help="SWI image name.")
+
+ops = ap.parse_args()
+
+if os.path.exists(ops.swi):
+    if ops.create and not ops.overwrite:
+        logger.critical("File '%s' exists." % ops.swi)
+        sys.exit(1)
+
+swi = None
+
+if ops.create or ops.overwrite:
+    if not ops.rootfs:
+        logger.critical("Rootfs required to create new SWI.")
+        sys.exit(1)
+
+    if not ops.manifest:
+        logger.critical("Manifest required to create new SWI.")
+        sys.exit(1)
+
+    swi = OnlSwitchImage(ops.swi, "w")
+    swi.add_rootfs(ops.rootfs)
+    swi.add_manifest(ops.manifest)
+    for f in ops.add_files:
+        swi.add(f, arcname=f)
+
+if swi is None:
+
+    if not os.path.exists(ops.swi):
+        logger.critical("SWI file %s does not exist." % ops.swi)
+        sys.exit(1)
+
+    swi = OnlSwitchImage(ops.swi, "r")
+
+if ops.contents:
+    print(" ".join(swi.get_contents()))
+
+if ops.platforms:
+    print(" ".join(swi.get_platforms()))