From bea9f68d0859d91a8201c0eae0cbcae58d8de7c4 Mon Sep 17 00:00:00 2001 From: Ivan Zorin Date: Tue, 1 Aug 2023 17:36:51 +0300 Subject: [PATCH 1/9] Implement CI target in Makefile to emulate github CI actions & artifacts --- Makefile | 53 +++++++++++++++++++++++++++++++++++---- scripts/IronOS.Dockerfile | 4 +-- source/metadata.py | 16 +++++++++++- 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 3c66495006..3ad915155d 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,14 @@ DOCKER_CMD=$(DOCKER_BIN) -f $(DOCKER_YML) run --rm builder # MkDocs config MKDOCS_YML=$(CURDIR)/scripts/IronOS-mkdocs.yml +# supported models +MODELS=TS100 TS80 TS80P Pinecil MHP30 Pinecilv2 S60 TS101 # target names & dir names +MODELS_ML=Pinecil Pinecilv2 # target names +MODELS_MULTILANG=Pinecil_multi-lang Pinecilv2_multi-lang # dir names + +# zip command (to pack artifacts) +ZIP=zip -q -j -r + ### targets @@ -91,7 +99,7 @@ list: @echo @echo "Full list of current supported IDs:" @echo " * LANG_ID: $(shell echo "`ls Translations/ | grep -e "^translation_.*.json$$" | sed -e 's,^translation_,,g; s,\.json$$,,g; ' | tr '\n' ' '`")" - @echo " * MODEL_ID: TS100 TS101 TS80 TS80P MHP30 Pinecil Pinecilv2 S60" + @echo " * MODEL_ID: $(MODELS)" @echo @echo "For example, to make a local build of IronOS firmware for TS100 with English language, just type:" @echo @@ -171,11 +179,41 @@ build-all: @chmod 0777 $(OUT_DIR) cd source && bash ./build.sh @echo "All Firmware built" - @cp -r $(OUT_HEX)/*.bin $(OUT_DIR) - @cp -r $(OUT_HEX)/*.hex $(OUT_DIR) - @cp -r $(OUT_HEX)/*.dfu $(OUT_DIR) + @for model in $(MODELS); do \ + mkdir -p $(OUT_DIR)/$${model} ; \ + cp -r $(OUT_HEX)/$${model}_*.bin $(OUT_DIR)/$${model}/ ; \ + cp -r $(OUT_HEX)/$${model}_*.hex $(OUT_DIR)/$${model}/ ; \ + cp -r $(OUT_HEX)/$${model}_*.dfu $(OUT_DIR)/$${model}/ ; \ + done; @echo "Resulting output directory: $(OUT_DIR)" +# target to build multilang supported builds for Pinecil & PinecilV2 +build-multilang: + @for modelml in $(MODELS_ML); do \ + $(MAKE) -C source/ -j2 model=$${modelml} firmware-multi_compressed_European firmware-multi_compressed_Bulgarian+Russian+Serbian+Ukrainian firmware-multi_Chinese+Japanese ; \ + mkdir -p $(OUT_DIR)/$${modelml}_multi-lang ; \ + cp $(OUT_HEX)/$${modelml}_multi_*.bin $(OUT_DIR)/$${modelml}_multi-lang ; \ + cp $(OUT_HEX)/$${modelml}_multi_*.hex $(OUT_DIR)/$${modelml}_multi-lang ; \ + cp $(OUT_HEX)/$${modelml}_multi_*.dfu $(OUT_DIR)/$${modelml}_multi-lang ; \ + done; + @echo "Resulting output directory: $(OUT_DIR)" + +# target to reproduce zips according to github CI settings; artifacts will be in $(OUT_DIR)/CI/*.zip +ci: tests build-all build-multilang + @mkdir -p $(OUT_DIR)/metadata; + @for m in $(MODELS) $(MODELS_MULTILANG); do \ + cp LICENSE scripts/LICENSE_RELEASE.md $(OUT_DIR)/$${m}/ ; \ + $(ZIP) $(OUT_DIR)/$${m}.zip $(OUT_DIR)/$${m} ; \ + ./source/metadata.py $${m}.json $${m}; \ + cp $(OUT_HEX)/$${m}.json $(OUT_DIR)/metadata; \ + done; + @$(ZIP) $(OUT_DIR)/metadata.zip $(OUT_DIR)/metadata + @mkdir -p $(OUT_DIR)/CI + @mv $(OUT_DIR)/*.zip $(OUT_DIR)/CI + @chmod 0777 $(OUT_DIR)/CI + @chmod 0666 $(OUT_DIR)/CI/*.zip + @echo "Resulting artifacts directory: $(OUT_DIR)/CI" + # pass-through target for Makefile inside source/ dir %: $(MAKE) -C source/ $@ @@ -191,4 +229,9 @@ clean-build: clean-full: clean-build docker-clean # phony targets -.PHONY: help list docker-check docker-shell docker-build docker-clean docs docs-deploy test-md test-sh test-py test-ccpp tests clean-build clean-full +.PHONY: help list +.PHONY: docker-check docker-shell docker-build docker-clean +.PHONY: docs docs-deploy +.PHONY: test-md test-sh test-py test-ccpp tests +.PHONY: build-all build-multilang ci +.PHONY: clean-build clean-full diff --git a/scripts/IronOS.Dockerfile b/scripts/IronOS.Dockerfile index 1543e1b028..c4514c1ab8 100644 --- a/scripts/IronOS.Dockerfile +++ b/scripts/IronOS.Dockerfile @@ -9,7 +9,7 @@ WORKDIR /build/ironos # Installing the two compilers (ARM & RISCV), python3 & pip, clang tools, etc.: ## - compilers: gcc-*, newlib-* ## - python3: py*, black (required to check Python code formatting) -## - misc: findutils, make, git, diffutils +## - misc: findutils, make, git, diffutils, zip ## - musl-dev (required for the multi lang firmwares) ## - clang (required for clang-format to check C++ code formatting) ## - shellcheck (to check sh scripts) @@ -17,7 +17,7 @@ WORKDIR /build/ironos ARG APK_COMPS="gcc-riscv-none-elf gcc-arm-none-eabi newlib-riscv-none-elf \ newlib-arm-none-eabi" ARG APK_PYTHON="python3 py3-pip black" -ARG APK_MISC="findutils make git diffutils" +ARG APK_MISC="findutils make git diffutils zip" ARG APK_DEV="musl-dev clang bash clang-extra-tools shellcheck" # PIP packages to check & test Python code diff --git a/source/metadata.py b/source/metadata.py index 775e2ecac4..4ce4717fc1 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -11,10 +11,18 @@ # This is used by automation like the Pinecil updater -if len(sys.argv) != 2: +if len(sys.argv) < 2: print("Requires the output json name as an arg") exit(1) +# Explicitly provide target file to scan for json output +ModelName = None +if len(sys.argv) == 3 and sys.argv[2] is not None and sys.argv[2] != "": + ModelName = sys.argv[2] + if ModelName == "Pinecil_multi-lang" or ModelName == "Pinecilv2_multi-lang": + # rename on-the-fly for direct compatibility with make target PINECILMODEL_multi-lang + ModelName = ModelName.rstrip("-lang") + HERE = Path(__file__).resolve().parent HexFileFolder = os.path.join(HERE, "Hexfile") @@ -61,6 +69,12 @@ def read_version(): if file_path.endswith(".hex") or file_path.endswith(".dfu"): # Find out what language this file is name: str = os.path.basename(file_path) + if ModelName is not None: + # If ModelName is provided as the second argument (compatible with make model=NAME fully) but current file name doesn't match the model name, then skip it + if not name.startswith(ModelName): + continue + if (ModelName == "Pinecil" or ModelName == "Pinecilv2") and not re.match(r"^" + ModelName + "_" + "([A-Z]+).*$", name): + continue matches = re.findall(r"^([a-zA-Z0-9]+)_(.+)\.(.+)$", name) if matches: matches = matches[0] From 75af90a74634c618f2c9e1c800fa44839f2d0e44 Mon Sep 17 00:00:00 2001 From: Ivan Zorin Date: Tue, 1 Aug 2023 21:53:44 +0300 Subject: [PATCH 2/9] Improve filter for metadata --- source/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/metadata.py b/source/metadata.py index 4ce4717fc1..8259437f6f 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -71,7 +71,7 @@ def read_version(): name: str = os.path.basename(file_path) if ModelName is not None: # If ModelName is provided as the second argument (compatible with make model=NAME fully) but current file name doesn't match the model name, then skip it - if not name.startswith(ModelName): + if not name.startswith(ModelName + "_"): continue if (ModelName == "Pinecil" or ModelName == "Pinecilv2") and not re.match(r"^" + ModelName + "_" + "([A-Z]+).*$", name): continue From 32946d976137bb83c2366b2471a1392421d03fab Mon Sep 17 00:00:00 2001 From: Ivan Zorin Date: Wed, 2 Aug 2023 16:10:12 +0300 Subject: [PATCH 3/9] metadata.py: update usage output for wrong number of input arguments / code review --- source/metadata.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/metadata.py b/source/metadata.py index 8259437f6f..7b3aff0e62 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -11,11 +11,13 @@ # This is used by automation like the Pinecil updater -if len(sys.argv) < 2: - print("Requires the output json name as an arg") +if len(sys.argv) < 2 or len(sys.argv) > 3: + print("Usage: metadata.py OUTPUT.json [model]") + print(" OUTPUT.json - the output json name with meta info about binary files") + print(" model [optional] - name of the model (as for `make model=NAME`) to scan files for explicitly (all files in source/Hexfile by default otherwise)") exit(1) -# Explicitly provide target file to scan for json output +# If model is provided explicitly to scan related files only for json output, then process the argument ModelName = None if len(sys.argv) == 3 and sys.argv[2] is not None and sys.argv[2] != "": ModelName = sys.argv[2] From bfa9fd0c6ac4c818087b00e73ab66e92da075f31 Mon Sep 17 00:00:00 2001 From: Ivan Zorin Date: Wed, 2 Aug 2023 16:12:44 +0300 Subject: [PATCH 4/9] metadata.py: remove excessive checks for the second input argument / code review --- source/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/metadata.py b/source/metadata.py index 7b3aff0e62..5ee4a52f94 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -19,7 +19,7 @@ # If model is provided explicitly to scan related files only for json output, then process the argument ModelName = None -if len(sys.argv) == 3 and sys.argv[2] is not None and sys.argv[2] != "": +if len(sys.argv) == 3: ModelName = sys.argv[2] if ModelName == "Pinecil_multi-lang" or ModelName == "Pinecilv2_multi-lang": # rename on-the-fly for direct compatibility with make target PINECILMODEL_multi-lang From 5efeb41cd9ba5c1a0dc79c0347813781e12ff4c2 Mon Sep 17 00:00:00 2001 From: Ivan Zorin Date: Wed, 2 Aug 2023 16:15:07 +0300 Subject: [PATCH 5/9] metadata.py: remove hard-coded model for multi-lang builds in ModelName argument processing / code review --- source/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/metadata.py b/source/metadata.py index 5ee4a52f94..eaef7fd211 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -21,8 +21,8 @@ ModelName = None if len(sys.argv) == 3: ModelName = sys.argv[2] - if ModelName == "Pinecil_multi-lang" or ModelName == "Pinecilv2_multi-lang": - # rename on-the-fly for direct compatibility with make target PINECILMODEL_multi-lang + if ModelName.endswith("_multi-lang"): + # rename on-the-fly for direct compatibility with make targets like PINECILMODEL_multi-lang ModelName = ModelName.rstrip("-lang") HERE = Path(__file__).resolve().parent From 8bd5105e5ec71d66beb168e7481eaac1b2ce868a Mon Sep 17 00:00:00 2001 From: Ivan Zorin Date: Wed, 2 Aug 2023 17:34:48 +0300 Subject: [PATCH 6/9] metadata.py: remove hard-coded models for multi-lang builds in file name pattern processing / code review --- source/metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/metadata.py b/source/metadata.py index eaef7fd211..8217cea5eb 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -75,7 +75,8 @@ def read_version(): # If ModelName is provided as the second argument (compatible with make model=NAME fully) but current file name doesn't match the model name, then skip it if not name.startswith(ModelName + "_"): continue - if (ModelName == "Pinecil" or ModelName == "Pinecilv2") and not re.match(r"^" + ModelName + "_" + "([A-Z]+).*$", name): + # If build of interest is not multi-lang one but scanning one is not MODEL_LANG-ID here, then skip it to avoid mess in json between MODEL_LANG-ID & MODEL_multi' + if not ModelName.endswith("_multi") and not re.match(r"^" + ModelName + "_" + "([A-Z]+).*$", name): continue matches = re.findall(r"^([a-zA-Z0-9]+)_(.+)\.(.+)$", name) if matches: From 7f94204ac302ab0b1fcb569f0bd262f5c13cba69 Mon Sep 17 00:00:00 2001 From: Ivan Zorin Date: Wed, 2 Aug 2023 17:39:48 +0300 Subject: [PATCH 7/9] metadata.py: update usage output to remove ambiguity about json extension for output file --- source/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/metadata.py b/source/metadata.py index 8217cea5eb..4332be12f4 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -12,8 +12,8 @@ if len(sys.argv) < 2 or len(sys.argv) > 3: - print("Usage: metadata.py OUTPUT.json [model]") - print(" OUTPUT.json - the output json name with meta info about binary files") + print("Usage: metadata.py OUTPUT_FILE [model]") + print(" OUTPUT_FILE - the name of output file in json format with meta info about binary files") print(" model [optional] - name of the model (as for `make model=NAME`) to scan files for explicitly (all files in source/Hexfile by default otherwise)") exit(1) From 6d0bfe0c07cc2983cc344046f81f30cda14f872d Mon Sep 17 00:00:00 2001 From: Ivan Zorin Date: Wed, 2 Aug 2023 17:41:12 +0300 Subject: [PATCH 8/9] metadata.py: unify new lines style formatting --- source/metadata.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/source/metadata.py b/source/metadata.py index 4332be12f4..5b49511637 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -10,7 +10,6 @@ # Creates an index metadata json file of the hexfiles folder # This is used by automation like the Pinecil updater - if len(sys.argv) < 2 or len(sys.argv) > 3: print("Usage: metadata.py OUTPUT_FILE [model]") print(" OUTPUT_FILE - the name of output file in json format with meta info about binary files") @@ -31,16 +30,13 @@ OutputJSONPath = os.path.join(HexFileFolder, sys.argv[1]) TranslationsFilesPath = os.path.join(HERE.parent, "Translations") - def load_json(filename: str): with open(filename) as f: return json.loads(f.read()) - def read_git_tag(): return f"{subprocess.check_output(['git', 'rev-parse', '--short=7', 'HEAD']).strip().decode('ascii').upper()}" - def read_version(): with open(HERE / "version.h") as version_file: for line in version_file: @@ -50,7 +46,6 @@ def read_version(): return matches[0] raise Exception("Could not parse version") - # Fetch our file listings translation_files = [os.path.join(TranslationsFilesPath, f) for f in os.listdir(TranslationsFilesPath) if os.path.isfile(os.path.join(TranslationsFilesPath, f)) and f.endswith(".json")] output_files = [os.path.join(HexFileFolder, f) for f in os.listdir(HexFileFolder) if os.path.isfile(os.path.join(HexFileFolder, f))] From c333ae8688689cd4456a8600630a474a111b6737 Mon Sep 17 00:00:00 2001 From: Ivan Zorin Date: Wed, 2 Aug 2023 17:57:47 +0300 Subject: [PATCH 9/9] metadata.py: sort the list of processing files in alphanumeric order before looping through them to get the same lang order on every generation in every json output file --- source/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/metadata.py b/source/metadata.py index 5b49511637..9700ed59b7 100755 --- a/source/metadata.py +++ b/source/metadata.py @@ -48,7 +48,7 @@ def read_version(): # Fetch our file listings translation_files = [os.path.join(TranslationsFilesPath, f) for f in os.listdir(TranslationsFilesPath) if os.path.isfile(os.path.join(TranslationsFilesPath, f)) and f.endswith(".json")] -output_files = [os.path.join(HexFileFolder, f) for f in os.listdir(HexFileFolder) if os.path.isfile(os.path.join(HexFileFolder, f))] +output_files = [os.path.join(HexFileFolder, f) for f in sorted(os.listdir(HexFileFolder)) if os.path.isfile(os.path.join(HexFileFolder, f))] parsed_languages = {} for path in translation_files: