From 11f6be223a874cf78c19c72be2b0f30c444130ca Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 10:52:35 -0400 Subject: [PATCH 01/54] Initial commit to kick an actual build. Does not do much, but it does fix a LGTM.com warning ... not totally useless! --- scripts/helpers/bloat_check.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index de9fa3b0de2dff..99f7a14fbfc422 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -172,7 +172,6 @@ def sendFileAsPrComment(job_name, filename, gh_token, gh_repo, gh_pr_number, """.format( title=titleHeading, table=compareTable, - jobName=job_name, rawReportText=rawText)) From 4e6cbe5fcc4eb868aeb5d1c3db6ff6fb375dc794 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 11:14:58 -0400 Subject: [PATCH 02/54] Starting to remove ci artifact fetch and to see if we can make github work --- .github/workflows/examples.yaml | 23 +++-- scripts/helpers/bloat_check.py | 44 ++++----- scripts/helpers/ci_fetch_artifacts.py | 124 -------------------------- 3 files changed, 34 insertions(+), 157 deletions(-) delete mode 100755 scripts/helpers/ci_fetch_artifacts.py diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 954a0924e10fc3..6dbc7a8a0aa442 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -31,18 +31,17 @@ jobs: run: mkdir -p example_binaries/$BUILD_TYPE-build; cp examples/wifi-echo/server/esp32/build/chip-wifi-echo.elf example_binaries/$BUILD_TYPE-build/chip-wifi-echo.elf - - # - name: Generate bloat report - # run: | - # mkdir -p "master_binaries/esp32-build" - # scripts/helpers/bloat_check.py \ - # --token "$GITHUB_TOKEN" \ - # --job "$GITHUB_WORKFLOW" \ - # --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ - # --build-output-dir "example_binaries/$BUILD_TYPE-build" \ - # --report-file bloat_report.txt \ - # --github-repository "$GITHUB_REPOSITORY" \ - # --github-comment-pr-number "$CIRCLE_PR_NUMBER" + - name: Generate bloat report + run: | + mkdir -p "master_binaries/esp32-build" + scripts/helpers/bloat_check.py \ + --token "$GITHUB_TOKEN" \ + --job "$GITHUB_WORKFLOW" \ + --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ + --build-output-dir "example_binaries/$BUILD_TYPE-build" \ + --report-file bloat_report.txt \ + --github-repository "$GITHUB_REPOSITORY" \ + --github-ref "$GITHUB_REF" - name: Copy aside bloat report & binaries run: | cp -r example_binaries/$BUILD_TYPE-build /tmp/output_binaries/ diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index 99f7a14fbfc422..af02c31832b476 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -27,8 +27,6 @@ import stat import subprocess -import ci_fetch_artifacts - class SectionChange: """Describes delta changes to a specific section""" @@ -203,10 +201,10 @@ def main(): parser.add_argument( '--github-repository', type=str, help='Repository to use for PR comments') parser.add_argument( - '--github-comment-pr-number', + '--github_ref', type=str, default=None, - help='To what PR to comment in github') + help='Github action ref, of format refs/pull/:prNumber/merge') parser.add_argument( '--log-level', default=logging.INFO, @@ -225,23 +223,27 @@ def main(): 'Required arguments missing. Please specify at least job and token.') return - try: - ci_fetch_artifacts.fetchArtifactsForJob(args.token, args.job, - args.artifact_download_dir) - except Exception as e: - logging.warning('Failed to fetch artifacts: %r', e) - - compareResults = generateBloatReport( - args.report_file, - args.artifact_download_dir, - args.build_output_dir, - title="Bloat report for job '%s'" % args.job) - - if args.github_api_token and args.github_repository and args.github_comment_pr_number: - sendFileAsPrComment(args.job, args.report_file, args.github_api_token, - args.github_repository, - int(args.github_comment_pr_number), compareResults) - + # TODO(andreilitvin): this needs refactoring and cleanup to fetch artifacts + # and process as needed + # + # try: + # ci_fetch_artifacts.fetchArtifactsForJob(args.token, args.job, + # args.artifact_download_dir) + # except Exception as e: + # logging.warning('Failed to fetch artifacts: %r', e) + # + # compareResults = generateBloatReport( + # args.report_file, + # args.artifact_download_dir, + # args.build_output_dir, + # title="Bloat report for job '%s'" % args.job) + # + # if args.github_api_token and args.github_repository and args.github_comment_pr_number: + # sendFileAsPrComment(args.job, args.report_file, args.github_api_token, + # args.github_repository, + # int(args.github_comment_pr_number), compareResults) + + logging.warning('NOT YET PORTED OVER/IMPLEMENTED') if __name__ == '__main__': # execute only if run as a script diff --git a/scripts/helpers/ci_fetch_artifacts.py b/scripts/helpers/ci_fetch_artifacts.py deleted file mode 100755 index 5cd4f5f7190ffd..00000000000000 --- a/scripts/helpers/ci_fetch_artifacts.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 - -# -# Copyright (c) 2020 Project CHIP Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -import attr -import coloredlogs -import logging - -from circleci.api import Api - - -@attr.s -class CompletedBuild: - """Represents basic build information""" - job_name = attr.ib(kw_only=True) - build_num = attr.ib(kw_only=True) - - -@attr.s -class ArtifactInfo: - """Represents basic artifact information""" - path = attr.ib(kw_only=True) # pretty path - url = attr.ib(kw_only=True) # download URL - - -class ApiHandler: - """Wrapper around circleci API calls, specific for artifact access.""" - - def __init__(self, token, user='project-chip', project='connectedhomeip'): - self.api = Api(token) - self.user = user - self.project = project - self.vcs_type = 'github' - - def fetchCompletedBuilds(self, branch='master'): - """Gets a snapshot of the latest successful builds in the specified branch.""" - for build in self.api.get_project_build_summary( - self.user, self.project, branch=branch, status_filter='successful'): - logging.debug('Fetched summary: %r', build) - yield CompletedBuild( - job_name=build['workflows']['job_name'], build_num=build['build_num']) - - def fetchLatestBuildNumber(self, job_name, branch='master'): - """Fetch the latest build number for the specified job name.""" - logging.info('Searching for the latest job "%s"', job_name) - - for build in self.fetchCompletedBuilds(branch): - logging.info('Found a completed build %r', build) - - if build.job_name == job_name: - logging.info('Found the latest job with id %d', build.build_num) - return build.build_num - - logging.error('No usable latest build was found.') - raise Exception('No build found for job "%s"' % job_name) - - def getArtifacts(self, build_num): - """Fetch artifacts for the specified build number.""" - for artifact in self.api.get_artifacts(self.user, self.project, build_num, - self.vcs_type): - logging.debug('Fetched artifact info: %r', artifact) - yield ArtifactInfo(path=artifact['path'], url=artifact['url']) - - -def fetchArtifactsForJob(token, job_name, download_dir): - """Download all job artifacts in to the specified directory.""" - # actual processing: downloads all the given artifacts - handler = ApiHandler(token) - for artifact in handler.getArtifacts( - handler.fetchLatestBuildNumber(job_name)): - logging.info('Downloading artifact %r...', artifact) - handler.api.download_artifact(artifact.url, download_dir) - logging.info('Download complete') - - -def main(): - """Main task if executed standalone.""" - parser = argparse.ArgumentParser(description='Fetch master build artifacts.') - parser.add_argument('--token', type=str, help='API token to use') - parser.add_argument('--job', type=str, help='What job to search for') - parser.add_argument( - '--download-dir', - type=str, - default='.', - help='Where to download the artifacts') - parser.add_argument( - '--log-level', - default=logging.INFO, - type=lambda x: getattr(logging, x), - help='Configure the logging level.') - args = parser.parse_args() - - # Ensures somewhat pretty logging of what is going on - logging.basicConfig( - level=args.log_level, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') - coloredlogs.install() - - if not args.token or not args.job: - logging.error( - 'Required arguments missing. Please specify at least job and token.') - return - - fetchArtifactsForJob(args.token, args.job, args.download_dir) - - -if __name__ == '__main__': - # execute only if run as a script - main() From c83cccba0386ecb434055e065c334f07baffe92a Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 11:29:47 -0400 Subject: [PATCH 03/54] Starting to parse and log refs to get pr numbers --- .vscode/settings.json | 2 +- scripts/helpers/bloat_check.py | 31 ++++++++++++++++++++++--------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 6788816b8a78fd..2510df9edd3170 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -154,4 +154,4 @@ ], "clang-format.fallbackStyle": "WebKit", "files.trimFinalNewlines": true -} \ No newline at end of file +} diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index af02c31832b476..f47c498d1363cb 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -17,13 +17,13 @@ # import argparse -import attr import coloredlogs import csv import github import io import logging import os +import re import stat import subprocess @@ -45,7 +45,8 @@ def __init__(self, name): self.sectionChanges = [] -SECTIONS_TO_WATCH = set(['.rodata', '.text', '.flash.rodata', '.flash.text', '.bss', '.data']) +SECTIONS_TO_WATCH = set( + ['.rodata', '.text', '.flash.rodata', '.flash.text', '.bss', '.data']) def filesInDirectory(dirName): @@ -151,8 +152,10 @@ def sendFileAsPrComment(job_name, filename, gh_token, gh_repo, gh_pr_number, compareTable = 'File | Section | File | VM\n---- | ---- | ----- | ---- \n' for file in compare_results: for change in file.sectionChanges: - compareTable += '{0} | {1} | {2} | {3}\n'.format( - file.fileName, change.section, change.fileChange, change.vmChange) + compareTable += '{0} | {1} | {2} | {3}\n'.format(file.fileName, + change.section, + change.fileChange, + change.vmChange) # NOTE: PRs are issues with attached patches, hence the API naming pull.create_issue_comment("""{title} @@ -167,10 +170,19 @@ def sendFileAsPrComment(job_name, filename, gh_token, gh_repo, gh_pr_number, ``` -""".format( - title=titleHeading, - table=compareTable, - rawReportText=rawText)) +""".format(title=titleHeading, table=compareTable, rawReportText=rawText)) + + +def extractPrNumberFromRef(refStr): + logging.info('EXTRACTING REF FROM "%s"', refStr) + match = re.compile('^refs/pull/(\\d*)/merge').match(refStr) + + if pattern: + return int(pattern.groups(1)) + + logging.warning('Cannot extract PR number from ref: "%s"', refStr) + + return None def main(): @@ -201,7 +213,7 @@ def main(): parser.add_argument( '--github-repository', type=str, help='Repository to use for PR comments') parser.add_argument( - '--github_ref', + '--github_ref', type=str, default=None, help='Github action ref, of format refs/pull/:prNumber/merge') @@ -245,6 +257,7 @@ def main(): logging.warning('NOT YET PORTED OVER/IMPLEMENTED') + if __name__ == '__main__': # execute only if run as a script main() From fd74808c119bb37783195928106f83e569b180b1 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 11:30:57 -0400 Subject: [PATCH 04/54] Fix argument format --- scripts/helpers/bloat_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index f47c498d1363cb..830f2ad3307e2a 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -213,7 +213,7 @@ def main(): parser.add_argument( '--github-repository', type=str, help='Repository to use for PR comments') parser.add_argument( - '--github_ref', + '--github-ref', type=str, default=None, help='Github action ref, of format refs/pull/:prNumber/merge') From d72a377637b5b8e1ed056baab951bb713e546a0c Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 11:34:08 -0400 Subject: [PATCH 05/54] Parse ref to get PR number --- scripts/helpers/bloat_check.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index 830f2ad3307e2a..153a3ed738177c 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -177,8 +177,8 @@ def extractPrNumberFromRef(refStr): logging.info('EXTRACTING REF FROM "%s"', refStr) match = re.compile('^refs/pull/(\\d*)/merge').match(refStr) - if pattern: - return int(pattern.groups(1)) + if match: + return int(match.group(1)) logging.warning('Cannot extract PR number from ref: "%s"', refStr) @@ -235,8 +235,11 @@ def main(): 'Required arguments missing. Please specify at least job and token.') return + comment_pr_number = extractPrNumberFromRef(args.github_ref) + # TODO(andreilitvin): this needs refactoring and cleanup to fetch artifacts # and process as needed + # # try: # ci_fetch_artifacts.fetchArtifactsForJob(args.token, args.job, @@ -250,10 +253,11 @@ def main(): # args.build_output_dir, # title="Bloat report for job '%s'" % args.job) # - # if args.github_api_token and args.github_repository and args.github_comment_pr_number: + if args.github_api_token and args.github_repository and comment_pr_number: + logging.warning('SEND REPORT DISABLED FOR NOW.') # sendFileAsPrComment(args.job, args.report_file, args.github_api_token, # args.github_repository, - # int(args.github_comment_pr_number), compareResults) + # comment_pr_number, compareResults) logging.warning('NOT YET PORTED OVER/IMPLEMENTED') From f534a36628c776d24763b1c67ccfe44d6856cd55 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 11:42:45 -0400 Subject: [PATCH 06/54] Log what exact argument is missing from a run --- scripts/helpers/bloat_check.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index 153a3ed738177c..daf0687c502d9f 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -230,9 +230,12 @@ def main(): format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') coloredlogs.install() - if not args.token or not args.job: - logging.error( - 'Required arguments missing. Please specify at least job and token.') + if not args.token: + logging.error('Required arguments missing: token is required.') + return + + if not args.job: + logging.error('Required arguments missing: job is required.') return comment_pr_number = extractPrNumberFromRef(args.github_ref) From 882d53cdec8631d09ed3a425deaf73a241af156a Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 11:49:47 -0400 Subject: [PATCH 07/54] Fix up job name and command line argument descriptions --- .github/workflows/examples.yaml | 2 +- scripts/helpers/bloat_check.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 6dbc7a8a0aa442..358fb14dbeca0d 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -36,7 +36,7 @@ jobs: mkdir -p "master_binaries/esp32-build" scripts/helpers/bloat_check.py \ --token "$GITHUB_TOKEN" \ - --job "$GITHUB_WORKFLOW" \ + --job "ESP32 Examples" \ --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index daf0687c502d9f..0d0769cebcdd11 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -190,7 +190,7 @@ def main(): parser = argparse.ArgumentParser(description='Fetch master build artifacts.') parser.add_argument('--token', type=str, help='API token to use') parser.add_argument( - '--job', type=str, help='From what job to fetch artifacts from') + '--job', type=str, help='Name of the job generating the report') parser.add_argument( '--artifact-download-dir', type=str, @@ -204,8 +204,8 @@ def main(): parser.add_argument( '--report-file', type=str, - default='report.txt', - help='From what job to fetch artifacts from') + default='report.csv', + help='Report file output name') parser.add_argument( '--github-api-token', type=str, From 307e13db1bb1485757315a32fbe19ff4a1b43578 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 12:00:44 -0400 Subject: [PATCH 08/54] Attempt to use github token from workflow secrets --- .github/workflows/examples.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 358fb14dbeca0d..2a3dcabb73636b 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -35,7 +35,7 @@ jobs: run: | mkdir -p "master_binaries/esp32-build" scripts/helpers/bloat_check.py \ - --token "$GITHUB_TOKEN" \ + --token "${{ secrets.GITHUB_TOKEN }}" \ --job "ESP32 Examples" \ --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ From 34fa58121708410652c8e53e9a0b75014746cd94 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 12:20:25 -0400 Subject: [PATCH 09/54] Pass in required tokens and add debug output --- .github/workflows/examples.yaml | 2 +- scripts/helpers/bloat_check.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 2a3dcabb73636b..96c78850e30fa6 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -35,11 +35,11 @@ jobs: run: | mkdir -p "master_binaries/esp32-build" scripts/helpers/bloat_check.py \ - --token "${{ secrets.GITHUB_TOKEN }}" \ --job "ESP32 Examples" \ --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ + --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ --github-repository "$GITHUB_REPOSITORY" \ --github-ref "$GITHUB_REF" - name: Copy aside bloat report & binaries diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index 0d0769cebcdd11..899b19ce91f06c 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -188,7 +188,6 @@ def extractPrNumberFromRef(refStr): def main(): """Main task if executed standalone.""" parser = argparse.ArgumentParser(description='Fetch master build artifacts.') - parser.add_argument('--token', type=str, help='API token to use') parser.add_argument( '--job', type=str, help='Name of the job generating the report') parser.add_argument( @@ -230,18 +229,28 @@ def main(): format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') coloredlogs.install() - if not args.token: - logging.error('Required arguments missing: token is required.') + if not args.github_api_token: + logging.error('Required arguments missing: github api token is required.') return if not args.job: logging.error('Required arguments missing: job is required.') return + if not args.github_ref: + logging.error('Required arguments missing: github_ref is required.') + return + comment_pr_number = extractPrNumberFromRef(args.github_ref) # TODO(andreilitvin): this needs refactoring and cleanup to fetch artifacts # and process as needed + ############################# DEBUG ############################### + for n in filter(lambda s: not s.startswith('_'), dir(args)): + if 'token' in n: + print("Keeping argument '%s' secret..." % (n, )) + else: + print("Received arg '%s': %r" % (n, getattr(args, n))) # # try: From 65217b9a7e8b453f66d31ab97fd7ebd8454e7507 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 12:46:19 -0400 Subject: [PATCH 10/54] Start with a github fetch api --- scripts/helpers/bloat_check.py | 26 +++++++++--------- scripts/helpers/github_fetch_artifacts.py | 32 +++++++++++++++++++++++ 2 files changed, 45 insertions(+), 13 deletions(-) create mode 100644 scripts/helpers/github_fetch_artifacts.py diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index 899b19ce91f06c..1421d4ba37a8ab 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -20,6 +20,7 @@ import coloredlogs import csv import github +import github_fetch_artifacts import io import logging import os @@ -252,19 +253,18 @@ def main(): else: print("Received arg '%s': %r" % (n, getattr(args, n))) - # - # try: - # ci_fetch_artifacts.fetchArtifactsForJob(args.token, args.job, - # args.artifact_download_dir) - # except Exception as e: - # logging.warning('Failed to fetch artifacts: %r', e) - # - # compareResults = generateBloatReport( - # args.report_file, - # args.artifact_download_dir, - # args.build_output_dir, - # title="Bloat report for job '%s'" % args.job) - # + try: + github_fetch_artifacts.fetchArtifactsForJob(args.job, args.github_api_token, args.github_repository, + args.artifact_download_dir) + except Exception as e: + logging.warning('Failed to fetch artifacts: %r', e) + + compareResults = generateBloatReport( + args.report_file, + args.artifact_download_dir, + args.build_output_dir, + title="Bloat report for job '%s'" % args.job) + if args.github_api_token and args.github_repository and comment_pr_number: logging.warning('SEND REPORT DISABLED FOR NOW.') # sendFileAsPrComment(args.job, args.report_file, args.github_api_token, diff --git a/scripts/helpers/github_fetch_artifacts.py b/scripts/helpers/github_fetch_artifacts.py new file mode 100644 index 00000000000000..a1b7cc7cd196e4 --- /dev/null +++ b/scripts/helpers/github_fetch_artifacts.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +# +# Copyright (c) 2020 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import github + +def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir): + + api = github.Github(githubToken) + repo = api.get_repo(githubRepo) + + logging.info('Fetching commits') + for c in repo.get_commits(): + logging.info('COMMIT: %r' % c) + + logging.error('NOT YET IMPLEMENTED') + pass \ No newline at end of file From cd61dfa3ac99f8564a9db955d2166e46cbc6067b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 12:57:48 -0400 Subject: [PATCH 11/54] Can now list master commit --- scripts/helpers/github_fetch_artifacts.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/helpers/github_fetch_artifacts.py b/scripts/helpers/github_fetch_artifacts.py index a1b7cc7cd196e4..aff73a82461800 100644 --- a/scripts/helpers/github_fetch_artifacts.py +++ b/scripts/helpers/github_fetch_artifacts.py @@ -16,17 +16,30 @@ # limitations under the License. # -import logging import github +import logging +import subprocess + + +def fetchMasterMergeCommitSHA(): + return subprocess.run( + 'git merge-base --fork-point master'.split(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ).stdout.decode('utf8').split()[0] + def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir): + masterCommitSHA = fetchMasterMergeCommitSHA() + + logging.info('Master merge commit: "%s"' % masterCommitSHA) api = github.Github(githubToken) repo = api.get_repo(githubRepo) - logging.info('Fetching commits') - for c in repo.get_commits(): - logging.info('COMMIT: %r' % c) + commit = repo.get_commit(masterCommitSHA) + + logging.info('COMMIT: %r' % commit) logging.error('NOT YET IMPLEMENTED') pass \ No newline at end of file From a1fae56c6a553629c17d0a3ad449b68d4d0fde45 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 14:10:05 -0400 Subject: [PATCH 12/54] Added ability to fetch artifacts ... we have a LOT of artifacts right now --- scripts/helpers/github_fetch_artifacts.py | 70 ++++++++++++++++++++--- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/scripts/helpers/github_fetch_artifacts.py b/scripts/helpers/github_fetch_artifacts.py index aff73a82461800..af7d6ddc17bc40 100644 --- a/scripts/helpers/github_fetch_artifacts.py +++ b/scripts/helpers/github_fetch_artifacts.py @@ -20,6 +20,54 @@ import logging import subprocess +# Artifacts can be fetched using: +# curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/project-chip/connectedhomeip/actions/artifacts + + +class ArtifactInfo(github.GithubObject.NonCompletableGithubObject): + + def _initAttributes(self): + pass + + def _useAttributes(self, attr): + if 'id' in attr: + self.id = self._makeIntAttribute(attr['id']).value + if 'node_id' in attr: + self.node_id = self._makeStringAttribute(attr['node_id']).value + if 'name' in attr: + self.name = self._makeStringAttribute(attr['name']).value + if 'size_in_bytes' in attr: + self.size_in_bytes = self._makeIntAttribute(attr['size_in_bytes']).value + if 'url' in attr: + self.url = self._makeStringAttribute(attr['url']).value + if 'archive_download_url' in attr: + self.url = self._makeStringAttribute(attr['archive_download_url']).value + if 'expired' in attr: + self.expired = self._makeBoolAttribute(attr['expired']).value + if 'created_at' in attr: + self.created_at = self._makeDatetimeAttribute(attr['created_at']).value + if 'updated_at' in attr: + self.expires_at = self._makeDatetimeAttribute(attr['updated_at']).value + + +class ArtifactFetcher(github.GithubObject.NonCompletableGithubObject): + + def __init__(self, repo): + self.url = repo.url + '/actions/artifacts' + self._requester = repo._requester + + logging.info('FETCHER: %r, %r', self.url, repo) + + def get_artifacts(self): + return github.PaginatedList.PaginatedList( + ArtifactInfo, + self._requester, + self.url, + None, + headers={'Accept': 'application/vnd.github.v3+json'}, + list_item='artifacts', + ) + def fetchMasterMergeCommitSHA(): return subprocess.run( @@ -30,16 +78,22 @@ def fetchMasterMergeCommitSHA(): def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir): - masterCommitSHA = fetchMasterMergeCommitSHA() + masterCommitSHA = fetchMasterMergeCommitSHA() + + logging.info('Master merge commit: "%s"', masterCommitSHA) - logging.info('Master merge commit: "%s"' % masterCommitSHA) + api = github.Github(githubToken) + repo = api.get_repo(githubRepo) - api = github.Github(githubToken) - repo = api.get_repo(githubRepo) + commit = repo.get_commit(masterCommitSHA) - commit = repo.get_commit(masterCommitSHA) + # We generally expect a single pull request in master for every such commit + for p in commit.get_pulls(): + logging.info('PULL: %r', p) - logging.info('COMMIT: %r' % commit) + fetcher = ArtifactFetcher(repo) + for idx, a in enumerate(fetcher.get_artifacts()): + logging.info('%d: Found artifact: %s from %r', idx, a.name, a.created_at) - logging.error('NOT YET IMPLEMENTED') - pass \ No newline at end of file + logging.error('NOT YET IMPLEMENTED') + pass From e1faea7d6689d91b562c7ac303520f0aeaa61dfe Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 14:15:41 -0400 Subject: [PATCH 13/54] When uploading artifacts, use the sha as suffix to make artifacts unique and easy to detect --- .github/workflows/examples.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 96c78850e30fa6..9a4642e40569d1 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -35,7 +35,7 @@ jobs: run: | mkdir -p "master_binaries/esp32-build" scripts/helpers/bloat_check.py \ - --job "ESP32 Examples" \ + --job "${{ env.BUILD_TYPE }} Examples" \ --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ @@ -54,7 +54,7 @@ jobs: - name: Uploading Binaries uses: actions/upload-artifact@v1 with: - name: ${{ env.BUILD_TYPE }}-example-build + name: ${{ env.BUILD_TYPE }}-example-build-${{ github.sha }} path: /tmp/output_binaries/${{ env.BUILD_TYPE }}-build nrf: From fadc7d5e96de03d12ea115f57aa19faa2e141cf0 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 14:18:02 -0400 Subject: [PATCH 14/54] Use contexts instead of environment variables --- .github/workflows/examples.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 9a4642e40569d1..9a3a6ca039c83b 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -40,8 +40,8 @@ jobs: --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ - --github-repository "$GITHUB_REPOSITORY" \ - --github-ref "$GITHUB_REF" + --github-repository "${{ github.repository }}" \ + --github-ref "${{ github.ref }}" - name: Copy aside bloat report & binaries run: | cp -r example_binaries/$BUILD_TYPE-build /tmp/output_binaries/ From 24e570f00f0ef40241d0684897a7325727dd36d6 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 14:32:09 -0400 Subject: [PATCH 15/54] Add debug for merge master info --- scripts/helpers/github_fetch_artifacts.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/helpers/github_fetch_artifacts.py b/scripts/helpers/github_fetch_artifacts.py index af7d6ddc17bc40..3fae00bc7b976a 100644 --- a/scripts/helpers/github_fetch_artifacts.py +++ b/scripts/helpers/github_fetch_artifacts.py @@ -70,11 +70,15 @@ def get_artifacts(self): def fetchMasterMergeCommitSHA(): - return subprocess.run( + result = subprocess.run( 'git merge-base --fork-point master'.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ).stdout.decode('utf8').split()[0] + ).stdout.decode('utf8') + + logging.info('Merge base result: "%r"' % result) + + return result.split()[0] def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir): From d0d245a7c2588f04ba5d3cba99d559c629defde8 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 14:37:28 -0400 Subject: [PATCH 16/54] Remove useless pass --- scripts/helpers/github_fetch_artifacts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/helpers/github_fetch_artifacts.py b/scripts/helpers/github_fetch_artifacts.py index 3fae00bc7b976a..ab9171a390eadc 100644 --- a/scripts/helpers/github_fetch_artifacts.py +++ b/scripts/helpers/github_fetch_artifacts.py @@ -100,4 +100,3 @@ def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir): logging.info('%d: Found artifact: %s from %r', idx, a.name, a.created_at) logging.error('NOT YET IMPLEMENTED') - pass From 6cc3357edc82d430d4b4ac0ca3bc9761ffebbac5 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 16:26:59 -0400 Subject: [PATCH 17/54] Start using a new docker image, which has a more recent git build --- .github/workflows/build.yaml | 5 +---- .github/workflows/examples.yaml | 5 +---- .github/workflows/qemu.yaml | 5 +---- .../docker/images/chip-build/Dockerfile | 19 +++++++++++++++++-- integrations/docker/images/chip-build/version | 2 +- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2ac1805483b637..e9843f3dba3d24 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,14 +13,11 @@ jobs: type: [main, clang, linux-embedded, mbedtls] env: BUILD_TYPE: ${{ matrix.type }} - BUILD_VERSION: 0.2.14 - BUILD_IMAGE: chip-build-openssl - BUILD_ORG: connectedhomeip runs-on: ubuntu-latest container: - image: "connectedhomeip/chip-build-openssl:0.2.14" + image: "connectedhomeip/chip-build-openssl:0.2.17" volumes: - "/tmp/log_output:/tmp/test_logs" diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 9a3a6ca039c83b..85b4269b6f49f8 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -10,14 +10,11 @@ jobs: env: BUILD_TYPE: esp32 - BUILD_VERSION: 0.2.14 - BUILD_IMAGE: chip-build-esp32 - BUILD_ORG: connectedhomeip runs-on: ubuntu-latest container: - image: connectedhomeip/chip-build-esp32:0.2.14 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION + image: connectedhomeip/chip-build-esp32:0.2.17 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" - "/tmp/output_binaries:/tmp/output_binaries" diff --git a/.github/workflows/qemu.yaml b/.github/workflows/qemu.yaml index 9353713cb6a20a..b3583227d92dae 100644 --- a/.github/workflows/qemu.yaml +++ b/.github/workflows/qemu.yaml @@ -10,14 +10,11 @@ jobs: env: BUILD_TYPE: esp32-qemu - BUILD_VERSION: 0.2.14 - BUILD_IMAGE: chip-build-esp32-qemu - BUILD_ORG: connectedhomeip runs-on: ubuntu-latest container: - image: connectedhomeip/chip-build-esp32-qemu:0.2.14 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION + image: connectedhomeip/chip-build-esp32-qemu:0.2.17 volumes: - "/tmp/log_output:/tmp/test_logs" diff --git a/integrations/docker/images/chip-build/Dockerfile b/integrations/docker/images/chip-build/Dockerfile index 1086e3bb343b29..c9ea2d98b06665 100644 --- a/integrations/docker/images/chip-build/Dockerfile +++ b/integrations/docker/images/chip-build/Dockerfile @@ -7,7 +7,6 @@ VOLUME "/var/source" RUN set -x \ && apt-get update \ && apt-get install -fy \ - git \ curl \ jq \ make \ @@ -24,6 +23,7 @@ RUN set -x \ libssl-dev \ unzip \ wget \ + libcurl4-openssl-dev \ libmbedtls-dev \ python-pip \ python3 \ @@ -33,6 +33,7 @@ RUN set -x \ libncursesw5-dev \ libnspr4-dev \ libxml2-dev \ + gettext \ flex \ bison \ gperf \ @@ -44,7 +45,7 @@ RUN set -x \ && : # last line RUN set -x \ - && pip3 install circleci attrs coloredlogs PyGithub \ + && pip3 install circleci attrs coloredlogs PyGithub pygit \ && : # last line # Install newer cmake toolchain @@ -60,6 +61,19 @@ RUN set -x \ && rm cmake-3.17.2.tar.gz \ && : # last line +# Install newer git version, to allow for github workflow checkout +RUN set -x \ + && wget https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.27.0.tar.gz \ + && tar xzf git-2.27.0.tar.gz \ + && cd git-2.27.0 \ + && ./configure --prefix=/usr \ + && make -j \ + && make install \ + && cd .. \ + && rm -rf git-2.27.0 \ + && rm git-2.27.0.tar.gz \ + && : # last line + # Install bloat comparison tools RUN set -x \ && git clone https://github.com/google/bloaty.git \ @@ -71,3 +85,4 @@ RUN set -x \ && cd ../.. \ && rm -rf bloaty \ && : # last line + diff --git a/integrations/docker/images/chip-build/version b/integrations/docker/images/chip-build/version index 9097948ffe8dc2..0fcb3bba7e0b87 100644 --- a/integrations/docker/images/chip-build/version +++ b/integrations/docker/images/chip-build/version @@ -1 +1 @@ -0.2.16 +0.2.17 From 3272ee8f802c4d0f2f52d39d7cf3a5d5ca9ba046 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 16:39:03 -0400 Subject: [PATCH 18/54] Revert build.yaml for now, apparently this is checked for. Awkward though- we should use the same docker image version --- .github/workflows/build.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e9843f3dba3d24..2ac1805483b637 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,11 +13,14 @@ jobs: type: [main, clang, linux-embedded, mbedtls] env: BUILD_TYPE: ${{ matrix.type }} + BUILD_VERSION: 0.2.14 + BUILD_IMAGE: chip-build-openssl + BUILD_ORG: connectedhomeip runs-on: ubuntu-latest container: - image: "connectedhomeip/chip-build-openssl:0.2.17" + image: "connectedhomeip/chip-build-openssl:0.2.14" volumes: - "/tmp/log_output:/tmp/test_logs" From 538fd246bb49c31a03fc13798b524914f64b944d Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 16:41:40 -0400 Subject: [PATCH 19/54] Try again to add workflow changes. I believe restyler is confused --- .github/workflows/build.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 2ac1805483b637..21551cf9c0a1eb 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,14 +13,11 @@ jobs: type: [main, clang, linux-embedded, mbedtls] env: BUILD_TYPE: ${{ matrix.type }} - BUILD_VERSION: 0.2.14 - BUILD_IMAGE: chip-build-openssl - BUILD_ORG: connectedhomeip runs-on: ubuntu-latest container: - image: "connectedhomeip/chip-build-openssl:0.2.14" + image: connectedhomeip/chip-build-openssl:0.2.17 volumes: - "/tmp/log_output:/tmp/test_logs" From 3f46aaa4f1d2f51b4cdd18a15a444f54fd50bade Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 16:43:19 -0400 Subject: [PATCH 20/54] Fix pattern for workflow changes (I hope) --- .restyled.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.restyled.yaml b/.restyled.yaml index 06c991894d01cc..656349c4f7efb8 100644 --- a/.restyled.yaml +++ b/.restyled.yaml @@ -39,7 +39,7 @@ request_review: none # Patterns to exclude from all Restylers exclude: - - ".github/workflows/**/*" # https://github.com/restyled-io/restyler/issues/73 + - ".github/workflows/*" # https://github.com/restyled-io/restyler/issues/73 - "third_party/**/*" - "third_party/**" - "third_party/*" From 2d3eccaf3babef28c40aa3ad407c0225b286a088 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 16:45:38 -0400 Subject: [PATCH 21/54] Kill YAML restyler - only restricted files have yaml (like github workflows) --- .restyled.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.restyled.yaml b/.restyled.yaml index 656349c4f7efb8..d408363794b696 100644 --- a/.restyled.yaml +++ b/.restyled.yaml @@ -66,7 +66,6 @@ restylers: include: - "**/Dockerfile" - "**/*.yml" - - "**/*.yaml" - "**/*.sh" - "**/*.c" - "**/*.cc" @@ -141,12 +140,6 @@ restylers: include: - "**/*.md" - "**/*.markdown" - - name: prettier-yaml - image: restyled/restyler-prettier:v1.19.1-2 - enabled: true - include: - - "**/*.yml" - - "**/*.yaml" - name: shellharden image: restyled/restyler-shellharden:v4.1.1-2 enabled: true From f126c81b0f2ea8aa6ab1651cca084d329ebf4a7c Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 16:46:29 -0400 Subject: [PATCH 22/54] Add final whitespace to build.yaml --- .github/workflows/build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 21551cf9c0a1eb..61b11390671484 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -74,3 +74,4 @@ jobs: # - name: Upload Code Coverage # if: ${{ contains('main', env.BUILD_TYPE) }} # run: bash <(curl -s https://codecov.io/bash) + From f6b63c6bd2fc16636c80251ba22d0e264bd95fef Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 16:49:05 -0400 Subject: [PATCH 23/54] reverted restyled.yaml. Something else is amiss and do not know what --- .restyled.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.restyled.yaml b/.restyled.yaml index d408363794b696..06c991894d01cc 100644 --- a/.restyled.yaml +++ b/.restyled.yaml @@ -39,7 +39,7 @@ request_review: none # Patterns to exclude from all Restylers exclude: - - ".github/workflows/*" # https://github.com/restyled-io/restyler/issues/73 + - ".github/workflows/**/*" # https://github.com/restyled-io/restyler/issues/73 - "third_party/**/*" - "third_party/**" - "third_party/*" @@ -66,6 +66,7 @@ restylers: include: - "**/Dockerfile" - "**/*.yml" + - "**/*.yaml" - "**/*.sh" - "**/*.c" - "**/*.cc" @@ -140,6 +141,12 @@ restylers: include: - "**/*.md" - "**/*.markdown" + - name: prettier-yaml + image: restyled/restyler-prettier:v1.19.1-2 + enabled: true + include: + - "**/*.yml" + - "**/*.yaml" - name: shellharden image: restyled/restyler-shellharden:v4.1.1-2 enabled: true From c846fff6b44ebd8ffa5e9816a594e14cb0071275 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 16:50:25 -0400 Subject: [PATCH 24/54] Added final newline to all workflow yaml files --- .github/workflows/examples.yaml | 1 + .github/workflows/qemu.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 85b4269b6f49f8..0ef2da23eaa4b6 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -110,3 +110,4 @@ jobs: mkdir -p example_binaries/$BUILD_TYPE cp examples/chip-tool/build/chip-standalone-demo.out \ example_binaries/$BUILD_TYPE/chip-standalone-demo.out + diff --git a/.github/workflows/qemu.yaml b/.github/workflows/qemu.yaml index b3583227d92dae..5d8367bea643bd 100644 --- a/.github/workflows/qemu.yaml +++ b/.github/workflows/qemu.yaml @@ -32,3 +32,4 @@ jobs: with: name: qemu-esp32-logs path: /tmp/log_output + From 17b1df9593b82cecc7740c0ec48515916a790438 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 16:58:25 -0400 Subject: [PATCH 25/54] Autoformatted build.yaml --- .github/workflows/build.yaml | 144 +++++++++++++++++------------------ 1 file changed, 70 insertions(+), 74 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 61b11390671484..055900d2d3181b 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,77 +1,73 @@ name: Builds - on: - push: - pull_request: - + push: + pull_request: jobs: - build: - name: Build - - strategy: - matrix: - type: [main, clang, linux-embedded, mbedtls] - env: - BUILD_TYPE: ${{ matrix.type }} - - runs-on: ubuntu-latest - - container: - image: connectedhomeip/chip-build-openssl:0.2.17 - volumes: - - "/tmp/log_output:/tmp/test_logs" - - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup Build Type - run: | - case $BUILD_TYPE in - "main") export BOOTSTRAP_ARGUMENTS="";; - "clang") export BOOTSTRAP_ARGUMENTS="CC=clang-9 CXX=clang++-9";; - "linux-embedded") export BOOTSTRAP_ARGUMENTS="--with-device-layer=linux";; - "mbedtls") export BOOTSTRAP_ARGUMENTS="--with-crypto=mbedtls";; - *) ;; - esac - - - name: Bootstrap - run: scripts/build/bootstrap.sh $BOOTSTRAP_ARGUMENTS - - name: Run Build - run: scripts/build/default.sh - - name: Run mbedTLS Tests - if: ${{ contains('mbedtls', env.BUILD_TYPE) }} - run: scripts/tests/mbedtls_tests.sh - - name: Run Crypto Tests - if: ${{ contains('mbedtls,clang,main', env.BUILD_TYPE) }} - run: scripts/tests/crypto_tests.sh - - name: Run Setup Payload Tests - if: ${{ contains('main,clang', env.BUILD_TYPE) }} - run: scripts/tests/setup_payload_tests.sh - - name: OpenSSL Tests - if: ${{ contains('main,clang', env.BUILD_TYPE) }} - run: scripts/tests/openssl_tests.sh - - name: Run Embedded Platform Tests - if: ${{ contains('linux-embedded', env.BUILD_TYPE) }} - run: make -C build/default/src/platform check - - name: Run Network Tests - run: scripts/tests/inet_tests.sh - - name: Run All Unit & Functional Tests - if: ${{ contains('main,clang', env.BUILD_TYPE) }} - run: scripts/tests/all_tests.sh - - name: Copying Logs Aside - run: scripts/tests/save_logs.sh /tmp/test_logs - - name: Uploading Logs - uses: actions/upload-artifact@v1 - with: - name: ${{ env.BUILD_TYPE }}-logs - path: /tmp/log_output - - name: Deployment Check - # if: ${{ contains('main', env.BUILD_TYPE) }} - run: scripts/build/distribution_check.sh - # - name: Run Code Coverage - # if: ${{ contains('main', env.BUILD_TYPE) }} - # run: scripts/tools/codecoverage.sh - # - name: Upload Code Coverage - # if: ${{ contains('main', env.BUILD_TYPE) }} - # run: bash <(curl -s https://codecov.io/bash) - + build: + name: Build + strategy: + matrix: + type: + - main + - clang + - linux-embedded + - mbedtls + env: + BUILD_TYPE: ${{ matrix.type }} + runs-on: ubuntu-latest + container: + image: connectedhomeip/chip-build-openssl:0.2.17 + volumes: + - /tmp/log_output:/tmp/test_logs + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Build Type + run: | + case $BUILD_TYPE in + "main") export BOOTSTRAP_ARGUMENTS="";; + "clang") export BOOTSTRAP_ARGUMENTS="CC=clang-9 CXX=clang++-9";; + "linux-embedded") export BOOTSTRAP_ARGUMENTS="--with-device-layer=linux";; + "mbedtls") export BOOTSTRAP_ARGUMENTS="--with-crypto=mbedtls";; + *) ;; + esac + - name: Bootstrap + run: scripts/build/bootstrap.sh $BOOTSTRAP_ARGUMENTS + - name: Run Build + run: scripts/build/default.sh + - name: Run mbedTLS Tests + if: ${{ contains('mbedtls', env.BUILD_TYPE) }} + run: scripts/tests/mbedtls_tests.sh + - name: Run Crypto Tests + if: ${{ contains('mbedtls,clang,main', env.BUILD_TYPE) }} + run: scripts/tests/crypto_tests.sh + - name: Run Setup Payload Tests + if: ${{ contains('main,clang', env.BUILD_TYPE) }} + run: scripts/tests/setup_payload_tests.sh + - name: OpenSSL Tests + if: ${{ contains('main,clang', env.BUILD_TYPE) }} + run: scripts/tests/openssl_tests.sh + - name: Run Embedded Platform Tests + if: ${{ contains('linux-embedded', env.BUILD_TYPE) }} + run: make -C build/default/src/platform check + - name: Run Network Tests + run: scripts/tests/inet_tests.sh + - name: Run All Unit & Functional Tests + if: ${{ contains('main,clang', env.BUILD_TYPE) }} + run: scripts/tests/all_tests.sh + - name: Copying Logs Aside + run: scripts/tests/save_logs.sh /tmp/test_logs + - name: Uploading Logs + uses: actions/upload-artifact@v1 + with: + name: ${{ env.BUILD_TYPE }}-logs + path: /tmp/log_output + - name: Deployment Check + # if: ${{ contains('main', env.BUILD_TYPE) }} + run: scripts/build/distribution_check.sh + # - name: Run Code Coverage + # if: ${{ contains('main', env.BUILD_TYPE) }} + # run: scripts/tools/codecoverage.sh + # - name: Upload Code Coverage + # if: ${{ contains('main', env.BUILD_TYPE) }} + # run: bash <(curl -s https://codecov.io/bash) From d0dd18395d70b984eaf0f7d896deae3e5038bb13 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 16:59:49 -0400 Subject: [PATCH 26/54] Revert build yaml format - this did not fix the restyler issue --- .github/workflows/build.yaml | 144 ++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 70 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 055900d2d3181b..61b11390671484 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -1,73 +1,77 @@ name: Builds + on: - push: - pull_request: + push: + pull_request: + jobs: - build: - name: Build - strategy: - matrix: - type: - - main - - clang - - linux-embedded - - mbedtls - env: - BUILD_TYPE: ${{ matrix.type }} - runs-on: ubuntu-latest - container: - image: connectedhomeip/chip-build-openssl:0.2.17 - volumes: - - /tmp/log_output:/tmp/test_logs - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Setup Build Type - run: | - case $BUILD_TYPE in - "main") export BOOTSTRAP_ARGUMENTS="";; - "clang") export BOOTSTRAP_ARGUMENTS="CC=clang-9 CXX=clang++-9";; - "linux-embedded") export BOOTSTRAP_ARGUMENTS="--with-device-layer=linux";; - "mbedtls") export BOOTSTRAP_ARGUMENTS="--with-crypto=mbedtls";; - *) ;; - esac - - name: Bootstrap - run: scripts/build/bootstrap.sh $BOOTSTRAP_ARGUMENTS - - name: Run Build - run: scripts/build/default.sh - - name: Run mbedTLS Tests - if: ${{ contains('mbedtls', env.BUILD_TYPE) }} - run: scripts/tests/mbedtls_tests.sh - - name: Run Crypto Tests - if: ${{ contains('mbedtls,clang,main', env.BUILD_TYPE) }} - run: scripts/tests/crypto_tests.sh - - name: Run Setup Payload Tests - if: ${{ contains('main,clang', env.BUILD_TYPE) }} - run: scripts/tests/setup_payload_tests.sh - - name: OpenSSL Tests - if: ${{ contains('main,clang', env.BUILD_TYPE) }} - run: scripts/tests/openssl_tests.sh - - name: Run Embedded Platform Tests - if: ${{ contains('linux-embedded', env.BUILD_TYPE) }} - run: make -C build/default/src/platform check - - name: Run Network Tests - run: scripts/tests/inet_tests.sh - - name: Run All Unit & Functional Tests - if: ${{ contains('main,clang', env.BUILD_TYPE) }} - run: scripts/tests/all_tests.sh - - name: Copying Logs Aside - run: scripts/tests/save_logs.sh /tmp/test_logs - - name: Uploading Logs - uses: actions/upload-artifact@v1 - with: - name: ${{ env.BUILD_TYPE }}-logs - path: /tmp/log_output - - name: Deployment Check - # if: ${{ contains('main', env.BUILD_TYPE) }} - run: scripts/build/distribution_check.sh - # - name: Run Code Coverage - # if: ${{ contains('main', env.BUILD_TYPE) }} - # run: scripts/tools/codecoverage.sh - # - name: Upload Code Coverage - # if: ${{ contains('main', env.BUILD_TYPE) }} - # run: bash <(curl -s https://codecov.io/bash) + build: + name: Build + + strategy: + matrix: + type: [main, clang, linux-embedded, mbedtls] + env: + BUILD_TYPE: ${{ matrix.type }} + + runs-on: ubuntu-latest + + container: + image: connectedhomeip/chip-build-openssl:0.2.17 + volumes: + - "/tmp/log_output:/tmp/test_logs" + + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Build Type + run: | + case $BUILD_TYPE in + "main") export BOOTSTRAP_ARGUMENTS="";; + "clang") export BOOTSTRAP_ARGUMENTS="CC=clang-9 CXX=clang++-9";; + "linux-embedded") export BOOTSTRAP_ARGUMENTS="--with-device-layer=linux";; + "mbedtls") export BOOTSTRAP_ARGUMENTS="--with-crypto=mbedtls";; + *) ;; + esac + + - name: Bootstrap + run: scripts/build/bootstrap.sh $BOOTSTRAP_ARGUMENTS + - name: Run Build + run: scripts/build/default.sh + - name: Run mbedTLS Tests + if: ${{ contains('mbedtls', env.BUILD_TYPE) }} + run: scripts/tests/mbedtls_tests.sh + - name: Run Crypto Tests + if: ${{ contains('mbedtls,clang,main', env.BUILD_TYPE) }} + run: scripts/tests/crypto_tests.sh + - name: Run Setup Payload Tests + if: ${{ contains('main,clang', env.BUILD_TYPE) }} + run: scripts/tests/setup_payload_tests.sh + - name: OpenSSL Tests + if: ${{ contains('main,clang', env.BUILD_TYPE) }} + run: scripts/tests/openssl_tests.sh + - name: Run Embedded Platform Tests + if: ${{ contains('linux-embedded', env.BUILD_TYPE) }} + run: make -C build/default/src/platform check + - name: Run Network Tests + run: scripts/tests/inet_tests.sh + - name: Run All Unit & Functional Tests + if: ${{ contains('main,clang', env.BUILD_TYPE) }} + run: scripts/tests/all_tests.sh + - name: Copying Logs Aside + run: scripts/tests/save_logs.sh /tmp/test_logs + - name: Uploading Logs + uses: actions/upload-artifact@v1 + with: + name: ${{ env.BUILD_TYPE }}-logs + path: /tmp/log_output + - name: Deployment Check + # if: ${{ contains('main', env.BUILD_TYPE) }} + run: scripts/build/distribution_check.sh + # - name: Run Code Coverage + # if: ${{ contains('main', env.BUILD_TYPE) }} + # run: scripts/tools/codecoverage.sh + # - name: Upload Code Coverage + # if: ${{ contains('main', env.BUILD_TYPE) }} + # run: bash <(curl -s https://codecov.io/bash) + From 63bb2e63b5dc8b173daa7aa86f8fd08d91dec485 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 17:08:41 -0400 Subject: [PATCH 27/54] Debug: figure out what branches exist in github workflows --- scripts/helpers/github_fetch_artifacts.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/helpers/github_fetch_artifacts.py b/scripts/helpers/github_fetch_artifacts.py index ab9171a390eadc..92a1099545348e 100644 --- a/scripts/helpers/github_fetch_artifacts.py +++ b/scripts/helpers/github_fetch_artifacts.py @@ -70,6 +70,12 @@ def get_artifacts(self): def fetchMasterMergeCommitSHA(): + logging.info('Available branches: %s' % subprocess.run( + 'git branch -la'.split(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ).stdout.decode('utf8')) + result = subprocess.run( 'git merge-base --fork-point master'.split(), stdout=subprocess.PIPE, From d34530653d75d37285ba7dcb33286fc0d5315bb4 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 17:37:58 -0400 Subject: [PATCH 28/54] Pass in the github head ref --- .github/workflows/examples.yaml | 3 ++- scripts/helpers/bloat_check.py | 7 ++++++- scripts/helpers/github_fetch_artifacts.py | 15 +++++---------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 0ef2da23eaa4b6..03f9649553a020 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -38,7 +38,8 @@ jobs: --report-file bloat_report.txt \ --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ --github-repository "${{ github.repository }}" \ - --github-ref "${{ github.ref }}" + --github-ref "${{ github.ref }}" \ + --git-master-ref "${{ github.head_ref }}" - name: Copy aside bloat report & binaries run: | cp -r example_binaries/$BUILD_TYPE-build /tmp/output_binaries/ diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index 1421d4ba37a8ab..80a98ea8082e38 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -222,6 +222,11 @@ def main(): default=logging.INFO, type=lambda x: getattr(logging, x), help='Configure the logging level.') + parser.add_argument( + '--git-master-ref', + type=str, + default=None, + help='Ref for fetching artifacts for the bloat report') args = parser.parse_args() # Ensures somewhat pretty logging of what is going on @@ -255,7 +260,7 @@ def main(): try: github_fetch_artifacts.fetchArtifactsForJob(args.job, args.github_api_token, args.github_repository, - args.artifact_download_dir) + args.artifact_download_dir, args.git_master_ref) except Exception as e: logging.warning('Failed to fetch artifacts: %r', e) diff --git a/scripts/helpers/github_fetch_artifacts.py b/scripts/helpers/github_fetch_artifacts.py index 92a1099545348e..2e1dcb52998628 100644 --- a/scripts/helpers/github_fetch_artifacts.py +++ b/scripts/helpers/github_fetch_artifacts.py @@ -70,12 +70,6 @@ def get_artifacts(self): def fetchMasterMergeCommitSHA(): - logging.info('Available branches: %s' % subprocess.run( - 'git branch -la'.split(), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ).stdout.decode('utf8')) - result = subprocess.run( 'git merge-base --fork-point master'.split(), stdout=subprocess.PIPE, @@ -87,15 +81,16 @@ def fetchMasterMergeCommitSHA(): return result.split()[0] -def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir): - masterCommitSHA = fetchMasterMergeCommitSHA() +def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir, compareRefSHA): + if not compareRefSHA: + compareRefSHA = fetchMasterMergeCommitSHA() - logging.info('Master merge commit: "%s"', masterCommitSHA) + logging.info('Master merge commit: "%s"', compareRefSHA) api = github.Github(githubToken) repo = api.get_repo(githubRepo) - commit = repo.get_commit(masterCommitSHA) + commit = repo.get_commit(compareRefSHA) # We generally expect a single pull request in master for every such commit for p in commit.get_pulls(): From c83cbf1b9acd0984c30f1c969d8f7b32b9599ab0 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 17:38:47 -0400 Subject: [PATCH 29/54] Correct the ref: use base ref --- .github/workflows/examples.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 03f9649553a020..e5a36a8f44e4e7 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -39,7 +39,7 @@ jobs: --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ --github-repository "${{ github.repository }}" \ --github-ref "${{ github.ref }}" \ - --git-master-ref "${{ github.head_ref }}" + --git-master-ref "${{ github.base_ref }}" - name: Copy aside bloat report & binaries run: | cp -r example_binaries/$BUILD_TYPE-build /tmp/output_binaries/ From c42ee390af5bef0aa967553f8c89a930cd3f4ea6 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 17:43:12 -0400 Subject: [PATCH 30/54] Use the 2.17 image for nrf and standalone as well --- .github/workflows/examples.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index e5a36a8f44e4e7..bdcf2446f4bd08 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -60,14 +60,11 @@ jobs: env: BUILD_TYPE: nrf - BUILD_VERSION: 0.2.14 - BUILD_IMAGE: chip-build-nrf-platform - BUILD_ORG: connectedhomeip runs-on: ubuntu-latest container: - image: connectedhomeip/chip-build-nrf-platform:0.2.14 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION + image: connectedhomeip/chip-build-nrf-platform:0.2.17 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" - "/tmp/output_binaries:/tmp/output_binaries" @@ -87,14 +84,11 @@ jobs: env: BUILD_TYPE: linux - BUILD_VERSION: 0.2.14 - BUILD_IMAGE: chip-build - BUILD_ORG: connectedhomeip runs-on: ubuntu-latest container: - image: connectedhomeip/chip-build:0.2.14 # $BUILD_IMAGE/$BUILD_ORG:$BUILD_VERSION + image: connectedhomeip/chip-build:0.2.17 volumes: - "/tmp/bloat_reports:/tmp/bloat_reports" - "/tmp/output_binaries:/tmp/output_binaries" From fbe930866e2451e3292e1a1497249d5134c0cbed Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Fri, 3 Jul 2020 17:49:47 -0400 Subject: [PATCH 31/54] Get a commit SHA from any rev compares --- scripts/helpers/github_fetch_artifacts.py | 34 ++++++++++++++--------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/scripts/helpers/github_fetch_artifacts.py b/scripts/helpers/github_fetch_artifacts.py index 2e1dcb52998628..1f773a433bf732 100644 --- a/scripts/helpers/github_fetch_artifacts.py +++ b/scripts/helpers/github_fetch_artifacts.py @@ -69,28 +69,36 @@ def get_artifacts(self): ) -def fetchMasterMergeCommitSHA(): - result = subprocess.run( - 'git merge-base --fork-point master'.split(), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ).stdout.decode('utf8') - - logging.info('Merge base result: "%r"' % result) +def fetchMasterMergeCommitSHA(revStr): + if revStr: + result = subprocess.run( + ['git', 'rev-parse', revStr], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ).stdout.decode('utf8') + + logging.info('Parsed revision %r base result: %r' % (revStr, result)) + else: + result = subprocess.run( + 'git merge-base --fork-point master'.split(), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ).stdout.decode('utf8') + + logging.info('Merge base result: %r' % result) return result.split()[0] -def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir, compareRefSHA): - if not compareRefSHA: - compareRefSHA = fetchMasterMergeCommitSHA() +def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir, compareRev): + masterMergeSha = fetchMasterMergeCommitSHA(compareRev) - logging.info('Master merge commit: "%s"', compareRefSHA) + logging.info('Master merge commit: "%s"', masterMergeSha) api = github.Github(githubToken) repo = api.get_repo(githubRepo) - commit = repo.get_commit(compareRefSHA) + commit = repo.get_commit(masterMergeSha) # We generally expect a single pull request in master for every such commit for p in commit.get_pulls(): From ebbca0fee5268b75d94fdbb524debcf56a9e666c Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 08:53:10 -0400 Subject: [PATCH 32/54] More attempts for debug --- .github/workflows/examples.yaml | 2 ++ examples/common/m5stack-tft/repo | 2 +- scripts/helpers/github_fetch_artifacts.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index bdcf2446f4bd08..9d58f915e108c9 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -28,6 +28,8 @@ jobs: run: mkdir -p example_binaries/$BUILD_TYPE-build; cp examples/wifi-echo/server/esp32/build/chip-wifi-echo.elf example_binaries/$BUILD_TYPE-build/chip-wifi-echo.elf + - name: DEBUG THROUGH SSH + uses: mxschmitt/action-tmate@v2 - name: Generate bloat report run: | mkdir -p "master_binaries/esp32-build" diff --git a/examples/common/m5stack-tft/repo b/examples/common/m5stack-tft/repo index 35be9760b4edf4..3a841faacb218d 160000 --- a/examples/common/m5stack-tft/repo +++ b/examples/common/m5stack-tft/repo @@ -1 +1 @@ -Subproject commit 35be9760b4edf485287f78c0cf652beb19c047f0 +Subproject commit 3a841faacb218dbdba2f54edb251d3a64d9833fb diff --git a/scripts/helpers/github_fetch_artifacts.py b/scripts/helpers/github_fetch_artifacts.py index 1f773a433bf732..62379b7d591f36 100644 --- a/scripts/helpers/github_fetch_artifacts.py +++ b/scripts/helpers/github_fetch_artifacts.py @@ -80,7 +80,7 @@ def fetchMasterMergeCommitSHA(revStr): logging.info('Parsed revision %r base result: %r' % (revStr, result)) else: result = subprocess.run( - 'git merge-base --fork-point master'.split(), + 'git merge-base --fork-point refs/heads/master'.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ).stdout.decode('utf8') From 2fa7efd95e84196cfd792253f3f94eef1b802727 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 09:08:36 -0400 Subject: [PATCH 33/54] Another debug attempt: tmate fails because no sudo --- .github/workflows/examples.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 9d58f915e108c9..1da6a320fd9eef 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -29,7 +29,16 @@ jobs: mkdir -p example_binaries/$BUILD_TYPE-build; cp examples/wifi-echo/server/esp32/build/chip-wifi-echo.elf example_binaries/$BUILD_TYPE-build/chip-wifi-echo.elf - name: DEBUG THROUGH SSH - uses: mxschmitt/action-tmate@v2 + run: curl -sL https://gist.githubusercontent.com/andy31415/22c43a3693a3401fd30d48aa2c3a5d77/raw/192b7ab5f5e49eed577836c33f6c7d58437a3dd7/SSHDebug.sh | bash + env: + # After sign up on the https://ngrok.com/ + # You can find this token here: https://dashboard.ngrok.com/get-started/setup + # NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} + NGROK_TOKEN: 1eJXREPXTREQuejZL8KLjauHDDd_2RqdxuPRaomU4WaPw6sY2 + + # This password you will use when authorizing via SSH + # USER_PASS: ${{ secrets.USER_PASS }} + USER_PASS: debugSSHtemp - name: Generate bloat report run: | mkdir -p "master_binaries/esp32-build" From ad210bf77e07325756c8b85feb8cc3fde366bdcc Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 09:28:37 -0400 Subject: [PATCH 34/54] Debug NRF: compiling esp is WAAAY to sllow --- .github/workflows/examples.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 1da6a320fd9eef..ecdc077c1c8825 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -28,17 +28,6 @@ jobs: run: mkdir -p example_binaries/$BUILD_TYPE-build; cp examples/wifi-echo/server/esp32/build/chip-wifi-echo.elf example_binaries/$BUILD_TYPE-build/chip-wifi-echo.elf - - name: DEBUG THROUGH SSH - run: curl -sL https://gist.githubusercontent.com/andy31415/22c43a3693a3401fd30d48aa2c3a5d77/raw/192b7ab5f5e49eed577836c33f6c7d58437a3dd7/SSHDebug.sh | bash - env: - # After sign up on the https://ngrok.com/ - # You can find this token here: https://dashboard.ngrok.com/get-started/setup - # NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} - NGROK_TOKEN: 1eJXREPXTREQuejZL8KLjauHDDd_2RqdxuPRaomU4WaPw6sY2 - - # This password you will use when authorizing via SSH - # USER_PASS: ${{ secrets.USER_PASS }} - USER_PASS: debugSSHtemp - name: Generate bloat report run: | mkdir -p "master_binaries/esp32-build" @@ -85,6 +74,17 @@ jobs: uses: actions/checkout@v2 - name: Build example nRF5 Lock App run: scripts/examples/nrf_lock_app.sh + - name: DEBUG THROUGH SSH + run: curl -sL https://gist.githubusercontent.com/andy31415/22c43a3693a3401fd30d48aa2c3a5d77/raw/59feefb48bc39581e74af89c3a2dc3762941d604/SSHDebug.sh | bash + env: + # After sign up on the https://ngrok.com/ + # You can find this token here: https://dashboard.ngrok.com/get-started/setup + # NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} + NGROK_TOKEN: 1eJXREPXTREQuejZL8KLjauHDDd_2RqdxuPRaomU4WaPw6sY2 + + # This password you will use when authorizing via SSH + # USER_PASS: ${{ secrets.USER_PASS }} + USER_PASS: debugSSHtemp - name: Preserve artifacts run: mkdir -p example_binaries/nrf-build ; cp From bf9d2c68108d5773f93743a852c9eb0cb42c2bc2 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 09:45:34 -0400 Subject: [PATCH 35/54] back to esp32 debug: need ngrok before build, since ngrok seems to run in background --- .github/workflows/examples.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index ecdc077c1c8825..3caae7d02e7edb 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -22,6 +22,17 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + - name: DEBUG THROUGH SSH + run: curl -sL https://gist.githubusercontent.com/andy31415/22c43a3693a3401fd30d48aa2c3a5d77/raw/59feefb48bc39581e74af89c3a2dc3762941d604/SSHDebug.sh | bash + env: + # After sign up on the https://ngrok.com/ + # You can find this token here: https://dashboard.ngrok.com/get-started/setup + # NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} + NGROK_TOKEN: 1eJXREPXTREQuejZL8KLjauHDDd_2RqdxuPRaomU4WaPw6sY2 + + # This password you will use when authorizing via SSH + # USER_PASS: ${{ secrets.USER_PASS }} + USER_PASS: debugSSHtemp - name: Build example Echo App run: scripts/examples/esp_echo_app.sh - name: Copy aside build products @@ -74,17 +85,6 @@ jobs: uses: actions/checkout@v2 - name: Build example nRF5 Lock App run: scripts/examples/nrf_lock_app.sh - - name: DEBUG THROUGH SSH - run: curl -sL https://gist.githubusercontent.com/andy31415/22c43a3693a3401fd30d48aa2c3a5d77/raw/59feefb48bc39581e74af89c3a2dc3762941d604/SSHDebug.sh | bash - env: - # After sign up on the https://ngrok.com/ - # You can find this token here: https://dashboard.ngrok.com/get-started/setup - # NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} - NGROK_TOKEN: 1eJXREPXTREQuejZL8KLjauHDDd_2RqdxuPRaomU4WaPw6sY2 - - # This password you will use when authorizing via SSH - # USER_PASS: ${{ secrets.USER_PASS }} - USER_PASS: debugSSHtemp - name: Preserve artifacts run: mkdir -p example_binaries/nrf-build ; cp From 419c410faf003c1912da03e4aead54632b0c9176 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 09:52:51 -0400 Subject: [PATCH 36/54] More debug - ssh fails completely, resorting to command line --- .github/workflows/examples.yaml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 3caae7d02e7edb..030a84db1ebc43 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -22,17 +22,10 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 - - name: DEBUG THROUGH SSH - run: curl -sL https://gist.githubusercontent.com/andy31415/22c43a3693a3401fd30d48aa2c3a5d77/raw/59feefb48bc39581e74af89c3a2dc3762941d604/SSHDebug.sh | bash - env: - # After sign up on the https://ngrok.com/ - # You can find this token here: https://dashboard.ngrok.com/get-started/setup - # NGROK_TOKEN: ${{ secrets.NGROK_TOKEN }} - NGROK_TOKEN: 1eJXREPXTREQuejZL8KLjauHDDd_2RqdxuPRaomU4WaPw6sY2 - - # This password you will use when authorizing via SSH - # USER_PASS: ${{ secrets.USER_PASS }} - USER_PASS: debugSSHtemp + - name: DEBUG OUTPUT + run: git branch -la + - name: DEBUG OUTPUT + run: git log --oneline -n 20 - name: Build example Echo App run: scripts/examples/esp_echo_app.sh - name: Copy aside build products From 4a0f2018fea55c7795742b1dd68bd8f3d3afe906 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 09:58:12 -0400 Subject: [PATCH 37/54] Add fetch depth to checkout --- .github/workflows/examples.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 030a84db1ebc43..037e37b9eef479 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -22,6 +22,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + with: + fetch-depth: 100 - name: DEBUG OUTPUT run: git branch -la - name: DEBUG OUTPUT From 72cf0f454c5c6344347e1efabd704be562bd04b4 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 10:02:48 -0400 Subject: [PATCH 38/54] checkout all history --- .github/workflows/examples.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 037e37b9eef479..4a3634103c834b 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -23,11 +23,13 @@ jobs: - name: Checkout uses: actions/checkout@v2 with: - fetch-depth: 100 + fetch-depth: 0 - name: DEBUG OUTPUT run: git branch -la - name: DEBUG OUTPUT run: git log --oneline -n 20 + - name: DEBUG OUTPUT + run: git merge-base --fork-point refs/heads/master - name: Build example Echo App run: scripts/examples/esp_echo_app.sh - name: Copy aside build products From 8808e58ac1c658e96bd114d066028c327cc6cfc7 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 10:06:24 -0400 Subject: [PATCH 39/54] Check if remotes master mergepoint works --- .github/workflows/examples.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 4a3634103c834b..011c9539654f46 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -29,7 +29,7 @@ jobs: - name: DEBUG OUTPUT run: git log --oneline -n 20 - name: DEBUG OUTPUT - run: git merge-base --fork-point refs/heads/master + run: git merge-base --fork-point remotes/origin/master - name: Build example Echo App run: scripts/examples/esp_echo_app.sh - name: Copy aside build products From b5d511eeb5a04d476f4e0149a11702746a190826 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 10:09:34 -0400 Subject: [PATCH 40/54] Log entire history, to check what the master rev looks like --- .github/workflows/examples.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 011c9539654f46..ec520288b048cf 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -27,7 +27,7 @@ jobs: - name: DEBUG OUTPUT run: git branch -la - name: DEBUG OUTPUT - run: git log --oneline -n 20 + run: git log --oneline -n 100 - name: DEBUG OUTPUT run: git merge-base --fork-point remotes/origin/master - name: Build example Echo App From 9fcdc4b18491ebf2bf58a4f41c6e232bcbb37f58 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 10:14:12 -0400 Subject: [PATCH 41/54] Update the script to use a command line fork point --- .github/workflows/examples.yaml | 2 +- scripts/helpers/bloat_check.py | 10 ++++++++-- scripts/helpers/github_fetch_artifacts.py | 9 +++++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index ec520288b048cf..a2faac41910bde 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -47,7 +47,7 @@ jobs: --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ --github-repository "${{ github.repository }}" \ --github-ref "${{ github.ref }}" \ - --git-master-ref "${{ github.base_ref }}" + --git-fork-poin "remotes/origin/${{ github.base_ref }}" - name: Copy aside bloat report & binaries run: | cp -r example_binaries/$BUILD_TYPE-build /tmp/output_binaries/ diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index 80a98ea8082e38..4a464e48a2a5bf 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -222,11 +222,16 @@ def main(): default=logging.INFO, type=lambda x: getattr(logging, x), help='Configure the logging level.') + parser.add_argument( + '--git-fork-point', + type=str, + default='refs/heads/master', + help='What forkpoint to get') parser.add_argument( '--git-master-ref', type=str, default=None, - help='Ref for fetching artifacts for the bloat report') + help='Ref for fetching artifacts for the bloat report (for debug purposes)') args = parser.parse_args() # Ensures somewhat pretty logging of what is going on @@ -260,7 +265,8 @@ def main(): try: github_fetch_artifacts.fetchArtifactsForJob(args.job, args.github_api_token, args.github_repository, - args.artifact_download_dir, args.git_master_ref) + args.artifact_download_dir, + args.git_fork_point, args.git_master_ref) except Exception as e: logging.warning('Failed to fetch artifacts: %r', e) diff --git a/scripts/helpers/github_fetch_artifacts.py b/scripts/helpers/github_fetch_artifacts.py index 62379b7d591f36..48e5f08ea9b55d 100644 --- a/scripts/helpers/github_fetch_artifacts.py +++ b/scripts/helpers/github_fetch_artifacts.py @@ -69,7 +69,7 @@ def get_artifacts(self): ) -def fetchMasterMergeCommitSHA(revStr): +def fetchMasterMergeCommitSHA(forkPointRef, revStr): if revStr: result = subprocess.run( ['git', 'rev-parse', revStr], @@ -79,8 +79,9 @@ def fetchMasterMergeCommitSHA(revStr): logging.info('Parsed revision %r base result: %r' % (revStr, result)) else: + logging.info('Finding merge point from %s' % forkPointRef) result = subprocess.run( - 'git merge-base --fork-point refs/heads/master'.split(), + ('git merge-base --fork-point %s' % forkPointRef).split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ).stdout.decode('utf8') @@ -90,8 +91,8 @@ def fetchMasterMergeCommitSHA(revStr): return result.split()[0] -def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir, compareRev): - masterMergeSha = fetchMasterMergeCommitSHA(compareRev) +def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir, forkPointRef, compareRev): + masterMergeSha = fetchMasterMergeCommitSHA(forkPointRef, compareRev) logging.info('Master merge commit: "%s"', masterMergeSha) From f046673e58ad43a61088fff98771ffcaf54b3509 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 10:19:46 -0400 Subject: [PATCH 42/54] Add depth 0 checkout for all items, remove debug output --- .github/workflows/examples.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index a2faac41910bde..c1093852b9e8df 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -22,14 +22,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + # Fetch depth 0 to get all history and be able to check mergepoint for bloat report with: fetch-depth: 0 - - name: DEBUG OUTPUT - run: git branch -la - - name: DEBUG OUTPUT - run: git log --oneline -n 100 - - name: DEBUG OUTPUT - run: git merge-base --fork-point remotes/origin/master - name: Build example Echo App run: scripts/examples/esp_echo_app.sh - name: Copy aside build products @@ -80,6 +75,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + # Fetch depth 0 to get all history and be able to check mergepoint for bloat report + with: + fetch-depth: 0 - name: Build example nRF5 Lock App run: scripts/examples/nrf_lock_app.sh - name: Preserve artifacts @@ -104,6 +102,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@v2 + # Fetch depth 0 to get all history and be able to check mergepoint for bloat report + with: + fetch-depth: 0 - name: Build example Standalone Echo Client run: scripts/examples/standalone_echo_client.sh - name: Build example Standalone Shell From ceca22c06b07e72e6390ec33c15e24dd32f1a070 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 11:05:47 -0400 Subject: [PATCH 43/54] Bloat report script seems to run --- .github/workflows/examples.yaml | 3 +- scripts/helpers/bloat_check.py | 21 +++-------- scripts/helpers/github_fetch_artifacts.py | 43 +++++++++++++++++------ 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index c1093852b9e8df..4d802f5bd95c17 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -32,10 +32,11 @@ jobs: mkdir -p example_binaries/$BUILD_TYPE-build; cp examples/wifi-echo/server/esp32/build/chip-wifi-echo.elf example_binaries/$BUILD_TYPE-build/chip-wifi-echo.elf - name: Generate bloat report + # job name MUST match prefix of artifact uploading (sans SHA) run: | mkdir -p "master_binaries/esp32-build" scripts/helpers/bloat_check.py \ - --job "${{ env.BUILD_TYPE }} Examples" \ + --job "${{ env.BUILD_TYPE }}-example-build" \ --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index 4a464e48a2a5bf..6d67df16bb0486 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -252,17 +252,6 @@ def main(): logging.error('Required arguments missing: github_ref is required.') return - comment_pr_number = extractPrNumberFromRef(args.github_ref) - - # TODO(andreilitvin): this needs refactoring and cleanup to fetch artifacts - # and process as needed - ############################# DEBUG ############################### - for n in filter(lambda s: not s.startswith('_'), dir(args)): - if 'token' in n: - print("Keeping argument '%s' secret..." % (n, )) - else: - print("Received arg '%s': %r" % (n, getattr(args, n))) - try: github_fetch_artifacts.fetchArtifactsForJob(args.job, args.github_api_token, args.github_repository, args.artifact_download_dir, @@ -276,13 +265,11 @@ def main(): args.build_output_dir, title="Bloat report for job '%s'" % args.job) + comment_pr_number = extractPrNumberFromRef(args.github_ref) if args.github_api_token and args.github_repository and comment_pr_number: - logging.warning('SEND REPORT DISABLED FOR NOW.') - # sendFileAsPrComment(args.job, args.report_file, args.github_api_token, - # args.github_repository, - # comment_pr_number, compareResults) - - logging.warning('NOT YET PORTED OVER/IMPLEMENTED') + sendFileAsPrComment(args.job, args.report_file, args.github_api_token, + args.github_repository, + comment_pr_number, compareResults) if __name__ == '__main__': diff --git a/scripts/helpers/github_fetch_artifacts.py b/scripts/helpers/github_fetch_artifacts.py index 48e5f08ea9b55d..a2ff871a958cc9 100644 --- a/scripts/helpers/github_fetch_artifacts.py +++ b/scripts/helpers/github_fetch_artifacts.py @@ -17,8 +17,11 @@ # import github +import io import logging +import requests import subprocess +import zipfile # Artifacts can be fetched using: # curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/project-chip/connectedhomeip/actions/artifacts @@ -41,7 +44,7 @@ def _useAttributes(self, attr): if 'url' in attr: self.url = self._makeStringAttribute(attr['url']).value if 'archive_download_url' in attr: - self.url = self._makeStringAttribute(attr['archive_download_url']).value + self.archive_download_url = self._makeStringAttribute(attr['archive_download_url']).value if 'expired' in attr: self.expired = self._makeBoolAttribute(attr['expired']).value if 'created_at' in attr: @@ -49,6 +52,19 @@ def _useAttributes(self, attr): if 'updated_at' in attr: self.expires_at = self._makeDatetimeAttribute(attr['updated_at']).value + def downloadBlob(self): + url = self.archive_download_url + logging.info('Fetching: %r' % url) + + headers, _ = self._requester.requestBlobAndCheck('GET', url) + + if headers['status'] != '302 Found': + raise Exception('Expected a redirect during blob download.') + + response = requests.get(headers['location']) + response.raise_for_status() + + return response.content class ArtifactFetcher(github.GithubObject.NonCompletableGithubObject): @@ -56,8 +72,6 @@ def __init__(self, repo): self.url = repo.url + '/actions/artifacts' self._requester = repo._requester - logging.info('FETCHER: %r, %r', self.url, repo) - def get_artifacts(self): return github.PaginatedList.PaginatedList( ArtifactInfo, @@ -99,14 +113,23 @@ def fetchArtifactsForJob(jobName, githubToken, githubRepo, downloadDir, forkPoin api = github.Github(githubToken) repo = api.get_repo(githubRepo) - commit = repo.get_commit(masterMergeSha) - - # We generally expect a single pull request in master for every such commit - for p in commit.get_pulls(): - logging.info('PULL: %r', p) + masterArtifactName = '%s-%s' % (jobName, masterMergeSha) + logging.info('Searching for artifact: %s' % masterArtifactName) + artifact = None fetcher = ArtifactFetcher(repo) for idx, a in enumerate(fetcher.get_artifacts()): - logging.info('%d: Found artifact: %s from %r', idx, a.name, a.created_at) + logging.debug('%d: Found artifact: %s from %r', idx, a.name, a.created_at) + + if a.name == masterArtifactName: + artifact = a + break + + if not artifact: + logging.error('Artifact not found') + return + + zipFile = zipfile.ZipFile(io.BytesIO(artifact.downloadBlob()), 'r') - logging.error('NOT YET IMPLEMENTED') + logging.info('Extracting zip file to %r' % downloadDir) + zipFile.extractall(downloadDir) From 5ca72c4071e7eb8c9f85b91c074a5815c5d3a14b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 11:10:09 -0400 Subject: [PATCH 44/54] Enable bloat report for nrf and linux as well --- .github/workflows/examples.yaml | 64 ++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 4d802f5bd95c17..bbbf22d47cc49a 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -29,12 +29,13 @@ jobs: run: scripts/examples/esp_echo_app.sh - name: Copy aside build products run: - mkdir -p example_binaries/$BUILD_TYPE-build; cp - examples/wifi-echo/server/esp32/build/chip-wifi-echo.elf example_binaries/$BUILD_TYPE-build/chip-wifi-echo.elf + mkdir -p example_binaries/$BUILD_TYPE-build + cp examples/wifi-echo/server/esp32/build/chip-wifi-echo.elf \ + example_binaries/$BUILD_TYPE-build/chip-wifi-echo.elf - name: Generate bloat report # job name MUST match prefix of artifact uploading (sans SHA) run: | - mkdir -p "master_binaries/esp32-build" + mkdir -p "master_binaries/$BUILD_TYPE-build" scripts/helpers/bloat_check.py \ --job "${{ env.BUILD_TYPE }}-example-build" \ --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ @@ -47,12 +48,6 @@ jobs: - name: Copy aside bloat report & binaries run: | cp -r example_binaries/$BUILD_TYPE-build /tmp/output_binaries/ - # cp bloat_report.txt /tmp/bloat_reports/ - # - name: Uploading Bloat Report - # uses: actions/upload-artifact@v1 - # with: - # name: ${{ env.BUILD_TYPE }}-bloat-report - # path: /tmp/bloat_reports/bloat_report.txt - name: Uploading Binaries uses: actions/upload-artifact@v1 with: @@ -81,10 +76,32 @@ jobs: fetch-depth: 0 - name: Build example nRF5 Lock App run: scripts/examples/nrf_lock_app.sh - - name: Preserve artifacts + - name: Copy aside build products run: - mkdir -p example_binaries/nrf-build ; cp - examples/lock-app/nrf5/build/chip-nrf52840-lock-example.out example_binaries/nrf-build/chip-nrf52840-lock-example.out + mkdir -p example_binaries/nrf-build + cp examples/lock-app/nrf5/build/chip-nrf52840-lock-example.out \ + example_binaries/$BUILD_TYPE-build/chip-nrf52840-lock-example.out + - name: Generate bloat report + # job name MUST match prefix of artifact uploading (sans SHA) + run: | + mkdir -p "master_binaries/$BUILD_TYPE-build" + scripts/helpers/bloat_check.py \ + --job "${{ env.BUILD_TYPE }}-example-build" \ + --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ + --build-output-dir "example_binaries/$BUILD_TYPE-build" \ + --report-file bloat_report.txt \ + --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ + --github-repository "${{ github.repository }}" \ + --github-ref "${{ github.ref }}" \ + --git-fork-poin "remotes/origin/${{ github.base_ref }}" + - name: Copy aside bloat report & binaries + run: | + cp -r example_binaries/$BUILD_TYPE-build /tmp/output_binaries/ + - name: Uploading Binaries + uses: actions/upload-artifact@v1 + with: + name: ${{ env.BUILD_TYPE }}-example-build-${{ github.sha }} + path: /tmp/output_binaries/${{ env.BUILD_TYPE }}-build linux-standalone: name: Linux Standalone @@ -114,5 +131,26 @@ jobs: run: | mkdir -p example_binaries/$BUILD_TYPE cp examples/chip-tool/build/chip-standalone-demo.out \ - example_binaries/$BUILD_TYPE/chip-standalone-demo.out + example_binaries/$BUILD_TYPE-build/chip-standalone-demo.out + - name: Generate bloat report + # job name MUST match prefix of artifact uploading (sans SHA) + run: | + mkdir -p "master_binaries/$BUILD_TYPE-build" + scripts/helpers/bloat_check.py \ + --job "${{ env.BUILD_TYPE }}-example-build" \ + --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ + --build-output-dir "example_binaries/$BUILD_TYPE-build" \ + --report-file bloat_report.txt \ + --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ + --github-repository "${{ github.repository }}" \ + --github-ref "${{ github.ref }}" \ + --git-fork-poin "remotes/origin/${{ github.base_ref }}" + - name: Copy aside bloat report & binaries + run: | + cp -r example_binaries/$BUILD_TYPE-build /tmp/output_binaries/ + - name: Uploading Binaries + uses: actions/upload-artifact@v1 + with: + name: ${{ env.BUILD_TYPE }}-example-build-${{ github.sha }} + path: /tmp/output_binaries/${{ env.BUILD_TYPE }}-build From 6f113749b9d9ad667f2c2556fdbde190f4fcc67e Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 11:16:00 -0400 Subject: [PATCH 45/54] Fix syntax for multiline run --- .github/workflows/examples.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index bbbf22d47cc49a..1e87339a3ceee7 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -28,7 +28,7 @@ jobs: - name: Build example Echo App run: scripts/examples/esp_echo_app.sh - name: Copy aside build products - run: + run: | mkdir -p example_binaries/$BUILD_TYPE-build cp examples/wifi-echo/server/esp32/build/chip-wifi-echo.elf \ example_binaries/$BUILD_TYPE-build/chip-wifi-echo.elf From 560772ab0b7823fe086b0b1307362c9d0633c50b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 11:21:31 -0400 Subject: [PATCH 46/54] another multiline fix --- .github/workflows/examples.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 1e87339a3ceee7..6cfa3c49abdce9 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -77,7 +77,7 @@ jobs: - name: Build example nRF5 Lock App run: scripts/examples/nrf_lock_app.sh - name: Copy aside build products - run: + run: | mkdir -p example_binaries/nrf-build cp examples/lock-app/nrf5/build/chip-nrf52840-lock-example.out \ example_binaries/$BUILD_TYPE-build/chip-nrf52840-lock-example.out From df8e7e77f9ae2b8ff2c1344eb189cc442cdd99c2 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 11:22:29 -0400 Subject: [PATCH 47/54] Fix output dir creation name for linux --- .github/workflows/examples.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 6cfa3c49abdce9..74b7a5d36ce973 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -129,7 +129,7 @@ jobs: run: scripts/examples/standalone_shell.sh - name: Preserve artifacts run: | - mkdir -p example_binaries/$BUILD_TYPE + mkdir -p example_binaries/$BUILD_TYPE-build cp examples/chip-tool/build/chip-standalone-demo.out \ example_binaries/$BUILD_TYPE-build/chip-standalone-demo.out - name: Generate bloat report From 5de43724065dc94e8a13eb877bebce98f8685c8b Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 11:27:41 -0400 Subject: [PATCH 48/54] Put a try/catch around PR comments: this fails until we update our API integration to use the chip bot secrets account --- scripts/helpers/bloat_check.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/helpers/bloat_check.py b/scripts/helpers/bloat_check.py index 6d67df16bb0486..474f1fe50fbc8a 100755 --- a/scripts/helpers/bloat_check.py +++ b/scripts/helpers/bloat_check.py @@ -267,9 +267,12 @@ def main(): comment_pr_number = extractPrNumberFromRef(args.github_ref) if args.github_api_token and args.github_repository and comment_pr_number: - sendFileAsPrComment(args.job, args.report_file, args.github_api_token, - args.github_repository, - comment_pr_number, compareResults) + try: + sendFileAsPrComment(args.job, args.report_file, args.github_api_token, + args.github_repository, + comment_pr_number, compareResults) + except Exception as e: + logging.warning('Failed to send PR comment: %r', e) if __name__ == '__main__': From 78d40a64586ff59405ade2b8c41d7d9d4af32fd3 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 12:04:39 -0400 Subject: [PATCH 49/54] Update token used for bloat reports --- .github/workflows/examples.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 74b7a5d36ce973..9c33b32a58c9d3 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -41,7 +41,7 @@ jobs: --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ - --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ + --github-api-token "${{ secrets.BLOAT_REPORT }}" \ --github-repository "${{ github.repository }}" \ --github-ref "${{ github.ref }}" \ --git-fork-poin "remotes/origin/${{ github.base_ref }}" @@ -90,7 +90,7 @@ jobs: --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ - --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ + --github-api-token "${{ secrets.BLOAT_REPORT }}" \ --github-repository "${{ github.repository }}" \ --github-ref "${{ github.ref }}" \ --git-fork-poin "remotes/origin/${{ github.base_ref }}" @@ -141,7 +141,7 @@ jobs: --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ - --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ + --github-api-token "${{ secrets.BLOAT_REPORT }}" \ --github-repository "${{ github.repository }}" \ --github-ref "${{ github.ref }}" \ --git-fork-poin "remotes/origin/${{ github.base_ref }}" From 2427b33c8e7684eaae4f760a4f6ab8a9cadb9863 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 12:19:10 -0400 Subject: [PATCH 50/54] Pass api key via env --- .github/workflows/examples.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 9c33b32a58c9d3..14d3e937ef275d 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -10,6 +10,7 @@ jobs: env: BUILD_TYPE: esp32 + GITHUB_API_KEY: ${{ secrets.BLOAT_REPORT }} runs-on: ubuntu-latest @@ -41,7 +42,7 @@ jobs: --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ - --github-api-token "${{ secrets.BLOAT_REPORT }}" \ + --github-api-token "$GITHUB_API_KEY" \ --github-repository "${{ github.repository }}" \ --github-ref "${{ github.ref }}" \ --git-fork-poin "remotes/origin/${{ github.base_ref }}" @@ -59,6 +60,7 @@ jobs: env: BUILD_TYPE: nrf + GITHUB_API_KEY: ${{ secrets.BLOAT_REPORT }} runs-on: ubuntu-latest @@ -90,7 +92,7 @@ jobs: --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ - --github-api-token "${{ secrets.BLOAT_REPORT }}" \ + --github-api-token "$GITHUB_API_KEY" \ --github-repository "${{ github.repository }}" \ --github-ref "${{ github.ref }}" \ --git-fork-poin "remotes/origin/${{ github.base_ref }}" @@ -108,6 +110,7 @@ jobs: env: BUILD_TYPE: linux + GITHUB_API_KEY: ${{ secrets.BLOAT_REPORT }} runs-on: ubuntu-latest @@ -141,7 +144,7 @@ jobs: --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ - --github-api-token "${{ secrets.BLOAT_REPORT }}" \ + --github-api-token "$GITHUB_API_KEY" \ --github-repository "${{ github.repository }}" \ --github-ref "${{ github.ref }}" \ --git-fork-poin "remotes/origin/${{ github.base_ref }}" From 7602666cd67205765386d70883b49364a52add38 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Mon, 6 Jul 2020 13:14:31 -0400 Subject: [PATCH 51/54] Use the restricted token for now --- .github/workflows/examples.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/examples.yaml b/.github/workflows/examples.yaml index 14d3e937ef275d..74b7a5d36ce973 100644 --- a/.github/workflows/examples.yaml +++ b/.github/workflows/examples.yaml @@ -10,7 +10,6 @@ jobs: env: BUILD_TYPE: esp32 - GITHUB_API_KEY: ${{ secrets.BLOAT_REPORT }} runs-on: ubuntu-latest @@ -42,7 +41,7 @@ jobs: --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ - --github-api-token "$GITHUB_API_KEY" \ + --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ --github-repository "${{ github.repository }}" \ --github-ref "${{ github.ref }}" \ --git-fork-poin "remotes/origin/${{ github.base_ref }}" @@ -60,7 +59,6 @@ jobs: env: BUILD_TYPE: nrf - GITHUB_API_KEY: ${{ secrets.BLOAT_REPORT }} runs-on: ubuntu-latest @@ -92,7 +90,7 @@ jobs: --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ - --github-api-token "$GITHUB_API_KEY" \ + --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ --github-repository "${{ github.repository }}" \ --github-ref "${{ github.ref }}" \ --git-fork-poin "remotes/origin/${{ github.base_ref }}" @@ -110,7 +108,6 @@ jobs: env: BUILD_TYPE: linux - GITHUB_API_KEY: ${{ secrets.BLOAT_REPORT }} runs-on: ubuntu-latest @@ -144,7 +141,7 @@ jobs: --artifact-download-dir "master_binaries/$BUILD_TYPE-build" \ --build-output-dir "example_binaries/$BUILD_TYPE-build" \ --report-file bloat_report.txt \ - --github-api-token "$GITHUB_API_KEY" \ + --github-api-token "${{ secrets.GITHUB_TOKEN }}" \ --github-repository "${{ github.repository }}" \ --github-ref "${{ github.ref }}" \ --git-fork-poin "remotes/origin/${{ github.base_ref }}" From 009553cb8a3d4bd1b07949ba146a45f10968ca65 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 9 Jul 2020 10:39:49 -0400 Subject: [PATCH 52/54] Bring back examples repos to make code compile --- examples/common/QRCode/repo | 1 + examples/common/m5stack-tft/repo | 1 + 2 files changed, 2 insertions(+) create mode 160000 examples/common/QRCode/repo create mode 160000 examples/common/m5stack-tft/repo diff --git a/examples/common/QRCode/repo b/examples/common/QRCode/repo new file mode 160000 index 00000000000000..08ac806145aa6a --- /dev/null +++ b/examples/common/QRCode/repo @@ -0,0 +1 @@ +Subproject commit 08ac806145aa6ab5d3e9df61d023092bb6cfe761 diff --git a/examples/common/m5stack-tft/repo b/examples/common/m5stack-tft/repo new file mode 160000 index 00000000000000..35be9760b4edf4 --- /dev/null +++ b/examples/common/m5stack-tft/repo @@ -0,0 +1 @@ +Subproject commit 35be9760b4edf485287f78c0cf652beb19c047f0 From 115ca34a5f3e5738671f01c7431cbfa53ec6d09a Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 9 Jul 2020 10:56:52 -0400 Subject: [PATCH 53/54] Undo changes to dockerfile and vscode --- .vscode/settings.json | 2 +- integrations/docker/images/chip-build/Dockerfile | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 2510df9edd3170..6788816b8a78fd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -154,4 +154,4 @@ ], "clang-format.fallbackStyle": "WebKit", "files.trimFinalNewlines": true -} +} \ No newline at end of file diff --git a/integrations/docker/images/chip-build/Dockerfile b/integrations/docker/images/chip-build/Dockerfile index 952b44c2f770c4..b6c5be6533d6ff 100644 --- a/integrations/docker/images/chip-build/Dockerfile +++ b/integrations/docker/images/chip-build/Dockerfile @@ -25,7 +25,6 @@ RUN set -x \ libssl-dev \ unzip \ wget \ - libcurl4-openssl-dev \ libmbedtls-dev \ python3 \ python3-pip \ @@ -34,7 +33,6 @@ RUN set -x \ libncursesw5-dev \ libnspr4-dev \ libxml2-dev \ - gettext \ flex \ bison \ gperf \ @@ -72,19 +70,6 @@ RUN set -x \ && rm -rf gn \ && : # last line -# Install newer git version, to allow for github workflow checkout -RUN set -x \ - && wget https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.27.0.tar.gz \ - && tar xzf git-2.27.0.tar.gz \ - && cd git-2.27.0 \ - && ./configure --prefix=/usr \ - && make -j \ - && make install \ - && cd .. \ - && rm -rf git-2.27.0 \ - && rm git-2.27.0.tar.gz \ - && : # last line - # Install bloat comparison tools RUN set -x \ && git clone https://github.com/google/bloaty.git \ @@ -96,4 +81,3 @@ RUN set -x \ && cd ../.. \ && rm -rf bloaty \ && : # last line - From 59daba0ccc2b70b31f9a55062f34bbab2b7d8b54 Mon Sep 17 00:00:00 2001 From: Andrei Litvin Date: Thu, 9 Jul 2020 11:08:40 -0400 Subject: [PATCH 54/54] add back m5stack-tft submodule --- .gitmodules | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitmodules b/.gitmodules index f696e3fdf55a84..dc76d02b80f28b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -34,6 +34,12 @@ branch = master ignore = dirty commit = 08ac806145aa6ab5d3e9df61d023092bb6cfe761 +[submodule "m5stack-tft"] + path = examples/common/m5stack-tft/repo + url = https://github.com/jeremyjh/ESP32_TFT_library.git + branch = master + ignore = dirty + commit = 35be9760b4edf485287f78c0cf652beb19c047f0 [submodule "pigweed"] path = third_party/pigweed/repo url = https://pigweed.googlesource.com/pigweed/pigweed