From 25e2a6588eced189915be18c6cac45e69c090bd1 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Fri, 9 Jul 2021 16:47:34 +0200 Subject: [PATCH 01/24] Clang-format hook --- .clang-format | 30 ++ tools/git-pre-commit | 22 ++ tools/lint/git-clang-format | 596 ++++++++++++++++++++++++++++++++++++ 3 files changed, 648 insertions(+) create mode 100644 .clang-format create mode 100644 tools/git-pre-commit create mode 100644 tools/lint/git-clang-format diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000000..30aec4f3150e --- /dev/null +++ b/.clang-format @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +Language: Cpp +BasedOnStyle: Google +ColumnLimit: 100 +#AlignAfterOpenBracket: AlwaysBreak +AlignConsecutiveAssignments: true +AlignConsecutiveDeclarations: false +AlignConsecutiveMacros: true +DerivePointerAlignment: false +SortIncludes: true +MaxEmptyLinesToKeep: 1 +PointerAlignment: Left +#Standard: Cpp11 \ No newline at end of file diff --git a/tools/git-pre-commit b/tools/git-pre-commit new file mode 100644 index 000000000000..2d697889af50 --- /dev/null +++ b/tools/git-pre-commit @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#!/bin/bash +set -e + +echo "Running pre-commit clang-format" +tools/lint/git-clang-format HEAD~ --force diff --git a/tools/lint/git-clang-format b/tools/lint/git-clang-format new file mode 100644 index 000000000000..0233ceb3a868 --- /dev/null +++ b/tools/lint/git-clang-format @@ -0,0 +1,596 @@ +#!/usr/bin/env python +# +#===- git-clang-format - ClangFormat Git Integration ---------*- python -*--===# +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +#===------------------------------------------------------------------------===# + +r""" +clang-format git integration +============================ + +This file provides a clang-format integration for git. Put it somewhere in your +path and ensure that it is executable. Then, "git clang-format" will invoke +clang-format on the changes in current files or a specific commit. + +For further details, run: +git clang-format -h + +Requires Python 2.7 or Python 3 +""" + +from __future__ import absolute_import, division, print_function +import argparse +import collections +import contextlib +import errno +import os +import re +import subprocess +import sys + +usage = 'git clang-format [OPTIONS] [] [] [--] [...]' + +desc = ''' +If zero or one commits are given, run clang-format on all lines that differ +between the working directory and , which defaults to HEAD. Changes are +only applied to the working directory. + +If two commits are given (requires --diff), run clang-format on all lines in the +second that differ from the first . + +The following git-config settings set the default of the corresponding option: + clangFormat.binary + clangFormat.commit + clangFormat.extensions + clangFormat.style +''' + +# Name of the temporary index file in which save the output of clang-format. +# This file is created within the .git directory. +temp_index_basename = 'clang-format-index' + + +Range = collections.namedtuple('Range', 'start, count') + + +def main(): + config = load_git_config() + + # In order to keep '--' yet allow options after positionals, we need to + # check for '--' ourselves. (Setting nargs='*' throws away the '--', while + # nargs=argparse.REMAINDER disallows options after positionals.) + argv = sys.argv[1:] + try: + idx = argv.index('--') + except ValueError: + dash_dash = [] + else: + dash_dash = argv[idx:] + argv = argv[:idx] + + default_extensions = ','.join([ + # From clang/lib/Frontend/FrontendOptions.cpp, all lower case + 'c', 'h', # C + 'm', # ObjC + 'mm', # ObjC++ + 'cc', 'cp', 'cpp', 'c++', 'cxx', 'hh', 'hpp', 'hxx', # C++ + 'cu', 'cuh', # CUDA + # Other languages that clang-format supports + 'proto', 'protodevel', # Protocol Buffers + 'java', # Java + 'js', # JavaScript + 'ts', # TypeScript + 'cs', # C Sharp + 'json', # Json + ]) + + p = argparse.ArgumentParser( + usage=usage, formatter_class=argparse.RawDescriptionHelpFormatter, + description=desc) + p.add_argument('--binary', + default=config.get('clangformat.binary', 'clang-format'), + help='path to clang-format'), + p.add_argument('--commit', + default=config.get('clangformat.commit', 'HEAD'), + help='default commit to use if none is specified'), + p.add_argument('--diff', action='store_true', + help='print a diff instead of applying the changes') + p.add_argument('--extensions', + default=config.get('clangformat.extensions', + default_extensions), + help=('comma-separated list of file extensions to format, ' + 'excluding the period and case-insensitive')), + p.add_argument('-f', '--force', action='store_true', + help='allow changes to unstaged files') + p.add_argument('-p', '--patch', action='store_true', + help='select hunks interactively') + p.add_argument('-q', '--quiet', action='count', default=0, + help='print less information') + p.add_argument('--style', + default=config.get('clangformat.style', None), + help='passed to clang-format'), + p.add_argument('-v', '--verbose', action='count', default=0, + help='print extra information') + # We gather all the remaining positional arguments into 'args' since we need + # to use some heuristics to determine whether or not was present. + # However, to print pretty messages, we make use of metavar and help. + p.add_argument('args', nargs='*', metavar='', + help='revision from which to compute the diff') + p.add_argument('ignored', nargs='*', metavar='...', + help='if specified, only consider differences in these files') + opts = p.parse_args(argv) + + opts.verbose -= opts.quiet + del opts.quiet + + commits, files = interpret_args(opts.args, dash_dash, opts.commit) + if len(commits) > 1: + if not opts.diff: + die('--diff is required when two commits are given') + else: + if len(commits) > 2: + die('at most two commits allowed; %d given' % len(commits)) + changed_lines = compute_diff_and_extract_lines(commits, files) + if opts.verbose >= 1: + ignored_files = set(changed_lines) + filter_by_extension(changed_lines, opts.extensions.lower().split(',')) + # The computed diff outputs absolute paths, so we must cd before accessing + # those files. + cd_to_toplevel() + filter_symlinks(changed_lines) + if opts.verbose >= 1: + ignored_files.difference_update(changed_lines) + if ignored_files: + print( + 'Ignoring changes in the following files (wrong extension or symlink):') + for filename in ignored_files: + print(' %s' % filename) + if changed_lines: + print('Running clang-format on the following files:') + for filename in changed_lines: + print(' %s' % filename) + if not changed_lines: + if opts.verbose >= 0: + print('no modified files to format') + return + if len(commits) > 1: + old_tree = commits[1] + new_tree = run_clang_format_and_save_to_tree(changed_lines, + revision=commits[1], + binary=opts.binary, + style=opts.style) + else: + old_tree = create_tree_from_workdir(changed_lines) + new_tree = run_clang_format_and_save_to_tree(changed_lines, + binary=opts.binary, + style=opts.style) + if opts.verbose >= 1: + print('old tree: %s' % old_tree) + print('new tree: %s' % new_tree) + if old_tree == new_tree: + if opts.verbose >= 0: + print('clang-format did not modify any files') + elif opts.diff: + print_diff(old_tree, new_tree) + else: + changed_files = apply_changes(old_tree, new_tree, force=opts.force, + patch_mode=opts.patch) + if (opts.verbose >= 0 and not opts.patch) or opts.verbose >= 1: + print('changed files:') + for filename in changed_files: + print(' %s' % filename) + + +def load_git_config(non_string_options=None): + """Return the git configuration as a dictionary. + + All options are assumed to be strings unless in `non_string_options`, in which + is a dictionary mapping option name (in lower case) to either "--bool" or + "--int".""" + if non_string_options is None: + non_string_options = {} + out = {} + for entry in run('git', 'config', '--list', '--null').split('\0'): + if entry: + if '\n' in entry: + name, value = entry.split('\n', 1) + else: + # A setting with no '=' ('\n' with --null) is implicitly 'true' + name = entry + value = 'true' + if name in non_string_options: + value = run('git', 'config', non_string_options[name], name) + out[name] = value + return out + + +def interpret_args(args, dash_dash, default_commit): + """Interpret `args` as "[commits] [--] [files]" and return (commits, files). + + It is assumed that "--" and everything that follows has been removed from + args and placed in `dash_dash`. + + If "--" is present (i.e., `dash_dash` is non-empty), the arguments to its + left (if present) are taken as commits. Otherwise, the arguments are checked + from left to right if they are commits or files. If commits are not given, + a list with `default_commit` is used.""" + if dash_dash: + if len(args) == 0: + commits = [default_commit] + else: + commits = args + for commit in commits: + object_type = get_object_type(commit) + if object_type not in ('commit', 'tag'): + if object_type is None: + die("'%s' is not a commit" % commit) + else: + die("'%s' is a %s, but a commit was expected" % (commit, object_type)) + files = dash_dash[1:] + elif args: + commits = [] + while args: + if not disambiguate_revision(args[0]): + break + commits.append(args.pop(0)) + if not commits: + commits = [default_commit] + files = args + else: + commits = [default_commit] + files = [] + return commits, files + + +def disambiguate_revision(value): + """Returns True if `value` is a revision, False if it is a file, or dies.""" + # If `value` is ambiguous (neither a commit nor a file), the following + # command will die with an appropriate error message. + run('git', 'rev-parse', value, verbose=False) + object_type = get_object_type(value) + if object_type is None: + return False + if object_type in ('commit', 'tag'): + return True + die('`%s` is a %s, but a commit or filename was expected' % + (value, object_type)) + + +def get_object_type(value): + """Returns a string description of an object's type, or None if it is not + a valid git object.""" + cmd = ['git', 'cat-file', '-t', value] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if p.returncode != 0: + return None + return convert_string(stdout.strip()) + + +def compute_diff_and_extract_lines(commits, files): + """Calls compute_diff() followed by extract_lines().""" + diff_process = compute_diff(commits, files) + changed_lines = extract_lines(diff_process.stdout) + diff_process.stdout.close() + diff_process.wait() + if diff_process.returncode != 0: + # Assume error was already printed to stderr. + sys.exit(2) + return changed_lines + + +def compute_diff(commits, files): + """Return a subprocess object producing the diff from `commits`. + + The return value's `stdin` file object will produce a patch with the + differences between the working directory and the first commit if a single + one was specified, or the difference between both specified commits, filtered + on `files` (if non-empty). Zero context lines are used in the patch.""" + git_tool = 'diff-index' + if len(commits) > 1: + git_tool = 'diff-tree' + cmd = ['git', git_tool, '-p', '-U0'] + commits + ['--'] + cmd.extend(files) + p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p.stdin.close() + return p + + +def extract_lines(patch_file): + """Extract the changed lines in `patch_file`. + + The return value is a dictionary mapping filename to a list of (start_line, + line_count) pairs. + + The input must have been produced with ``-U0``, meaning unidiff format with + zero lines of context. The return value is a dict mapping filename to a + list of line `Range`s.""" + matches = {} + for line in patch_file: + line = convert_string(line) + match = re.search(r'^\+\+\+\ [^/]+/(.*)', line) + if match: + filename = match.group(1).rstrip('\r\n') + match = re.search(r'^@@ -[0-9,]+ \+(\d+)(,(\d+))?', line) + if match: + start_line = int(match.group(1)) + line_count = 1 + if match.group(3): + line_count = int(match.group(3)) + if line_count > 0: + matches.setdefault(filename, []).append(Range(start_line, line_count)) + return matches + + +def filter_by_extension(dictionary, allowed_extensions): + """Delete every key in `dictionary` that doesn't have an allowed extension. + + `allowed_extensions` must be a collection of lowercase file extensions, + excluding the period.""" + allowed_extensions = frozenset(allowed_extensions) + for filename in list(dictionary.keys()): + base_ext = filename.rsplit('.', 1) + if len(base_ext) == 1 and '' in allowed_extensions: + continue + if len(base_ext) == 1 or base_ext[1].lower() not in allowed_extensions: + del dictionary[filename] + + +def filter_symlinks(dictionary): + """Delete every key in `dictionary` that is a symlink.""" + for filename in list(dictionary.keys()): + if os.path.islink(filename): + del dictionary[filename] + + +def cd_to_toplevel(): + """Change to the top level of the git repository.""" + toplevel = run('git', 'rev-parse', '--show-toplevel') + os.chdir(toplevel) + + +def create_tree_from_workdir(filenames): + """Create a new git tree with the given files from the working directory. + + Returns the object ID (SHA-1) of the created tree.""" + return create_tree(filenames, '--stdin') + + +def run_clang_format_and_save_to_tree(changed_lines, revision=None, + binary='clang-format', style=None): + """Run clang-format on each file and save the result to a git tree. + + Returns the object ID (SHA-1) of the created tree.""" + def iteritems(container): + try: + return container.iteritems() # Python 2 + except AttributeError: + return container.items() # Python 3 + def index_info_generator(): + for filename, line_ranges in iteritems(changed_lines): + if revision: + git_metadata_cmd = ['git', 'ls-tree', + '%s:%s' % (revision, os.path.dirname(filename)), + os.path.basename(filename)] + git_metadata = subprocess.Popen(git_metadata_cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + stdout = git_metadata.communicate()[0] + mode = oct(int(stdout.split()[0], 8)) + else: + mode = oct(os.stat(filename).st_mode) + # Adjust python3 octal format so that it matches what git expects + if mode.startswith('0o'): + mode = '0' + mode[2:] + blob_id = clang_format_to_blob(filename, line_ranges, + revision=revision, + binary=binary, + style=style) + yield '%s %s\t%s' % (mode, blob_id, filename) + return create_tree(index_info_generator(), '--index-info') + + +def create_tree(input_lines, mode): + """Create a tree object from the given input. + + If mode is '--stdin', it must be a list of filenames. If mode is + '--index-info' is must be a list of values suitable for "git update-index + --index-info", such as " ". Any other mode + is invalid.""" + assert mode in ('--stdin', '--index-info') + cmd = ['git', 'update-index', '--add', '-z', mode] + with temporary_index_file(): + p = subprocess.Popen(cmd, stdin=subprocess.PIPE) + for line in input_lines: + p.stdin.write(to_bytes('%s\0' % line)) + p.stdin.close() + if p.wait() != 0: + die('`%s` failed' % ' '.join(cmd)) + tree_id = run('git', 'write-tree') + return tree_id + + +def clang_format_to_blob(filename, line_ranges, revision=None, + binary='clang-format', style=None): + """Run clang-format on the given file and save the result to a git blob. + + Runs on the file in `revision` if not None, or on the file in the working + directory if `revision` is None. + + Returns the object ID (SHA-1) of the created blob.""" + clang_format_cmd = [binary] + if style: + clang_format_cmd.extend(['-style='+style]) + clang_format_cmd.extend([ + '-lines=%s:%s' % (start_line, start_line+line_count-1) + for start_line, line_count in line_ranges]) + if revision: + clang_format_cmd.extend(['-assume-filename='+filename]) + git_show_cmd = ['git', 'cat-file', 'blob', '%s:%s' % (revision, filename)] + git_show = subprocess.Popen(git_show_cmd, stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + git_show.stdin.close() + clang_format_stdin = git_show.stdout + else: + clang_format_cmd.extend([filename]) + git_show = None + clang_format_stdin = subprocess.PIPE + try: + clang_format = subprocess.Popen(clang_format_cmd, stdin=clang_format_stdin, + stdout=subprocess.PIPE) + if clang_format_stdin == subprocess.PIPE: + clang_format_stdin = clang_format.stdin + except OSError as e: + if e.errno == errno.ENOENT: + die('cannot find executable "%s"' % binary) + else: + raise + clang_format_stdin.close() + hash_object_cmd = ['git', 'hash-object', '-w', '--path='+filename, '--stdin'] + hash_object = subprocess.Popen(hash_object_cmd, stdin=clang_format.stdout, + stdout=subprocess.PIPE) + clang_format.stdout.close() + stdout = hash_object.communicate()[0] + if hash_object.returncode != 0: + die('`%s` failed' % ' '.join(hash_object_cmd)) + if clang_format.wait() != 0: + die('`%s` failed' % ' '.join(clang_format_cmd)) + if git_show and git_show.wait() != 0: + die('`%s` failed' % ' '.join(git_show_cmd)) + return convert_string(stdout).rstrip('\r\n') + + +@contextlib.contextmanager +def temporary_index_file(tree=None): + """Context manager for setting GIT_INDEX_FILE to a temporary file and deleting + the file afterward.""" + index_path = create_temporary_index(tree) + old_index_path = os.environ.get('GIT_INDEX_FILE') + os.environ['GIT_INDEX_FILE'] = index_path + try: + yield + finally: + if old_index_path is None: + del os.environ['GIT_INDEX_FILE'] + else: + os.environ['GIT_INDEX_FILE'] = old_index_path + os.remove(index_path) + + +def create_temporary_index(tree=None): + """Create a temporary index file and return the created file's path. + + If `tree` is not None, use that as the tree to read in. Otherwise, an + empty index is created.""" + gitdir = run('git', 'rev-parse', '--git-dir') + path = os.path.join(gitdir, temp_index_basename) + if tree is None: + tree = '--empty' + run('git', 'read-tree', '--index-output='+path, tree) + return path + + +def print_diff(old_tree, new_tree): + """Print the diff between the two trees to stdout.""" + # We use the porcelain 'diff' and not plumbing 'diff-tree' because the output + # is expected to be viewed by the user, and only the former does nice things + # like color and pagination. + # + # We also only print modified files since `new_tree` only contains the files + # that were modified, so unmodified files would show as deleted without the + # filter. + subprocess.check_call(['git', 'diff', '--diff-filter=M', old_tree, new_tree, + '--']) + + +def apply_changes(old_tree, new_tree, force=False, patch_mode=False): + """Apply the changes in `new_tree` to the working directory. + + Bails if there are local changes in those files and not `force`. If + `patch_mode`, runs `git checkout --patch` to select hunks interactively.""" + changed_files = run('git', 'diff-tree', '--diff-filter=M', '-r', '-z', + '--name-only', old_tree, + new_tree).rstrip('\0').split('\0') + if not force: + unstaged_files = run('git', 'diff-files', '--name-status', *changed_files) + if unstaged_files: + print('The following files would be modified but ' + 'have unstaged changes:', file=sys.stderr) + print(unstaged_files, file=sys.stderr) + print('Please commit, stage, or stash them first.', file=sys.stderr) + sys.exit(2) + if patch_mode: + # In patch mode, we could just as well create an index from the new tree + # and checkout from that, but then the user will be presented with a + # message saying "Discard ... from worktree". Instead, we use the old + # tree as the index and checkout from new_tree, which gives the slightly + # better message, "Apply ... to index and worktree". This is not quite + # right, since it won't be applied to the user's index, but oh well. + with temporary_index_file(old_tree): + subprocess.check_call(['git', 'checkout', '--patch', new_tree]) + index_tree = old_tree + else: + with temporary_index_file(new_tree): + run('git', 'checkout-index', '-a', '-f') + return changed_files + + +def run(*args, **kwargs): + stdin = kwargs.pop('stdin', '') + verbose = kwargs.pop('verbose', True) + strip = kwargs.pop('strip', True) + for name in kwargs: + raise TypeError("run() got an unexpected keyword argument '%s'" % name) + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + stdout, stderr = p.communicate(input=stdin) + + stdout = convert_string(stdout) + stderr = convert_string(stderr) + + if p.returncode == 0: + if stderr: + if verbose: + print('`%s` printed to stderr:' % ' '.join(args), file=sys.stderr) + print(stderr.rstrip(), file=sys.stderr) + if strip: + stdout = stdout.rstrip('\r\n') + return stdout + if verbose: + print('`%s` returned %s' % (' '.join(args), p.returncode), file=sys.stderr) + if stderr: + print(stderr.rstrip(), file=sys.stderr) + sys.exit(2) + + +def die(message): + print('error:', message, file=sys.stderr) + sys.exit(2) + + +def to_bytes(str_input): + # Encode to UTF-8 to get binary data. + if isinstance(str_input, bytes): + return str_input + return str_input.encode('utf-8') + + +def to_string(bytes_input): + if isinstance(bytes_input, str): + return bytes_input + return bytes_input.encode('utf-8') + + +def convert_string(bytes_input): + try: + return to_string(bytes_input.decode('utf-8')) + except AttributeError: # 'str' object has no attribute 'decode'. + return str(bytes_input) + except UnicodeError: + return str(bytes_input) + +if __name__ == '__main__': + main() From 813070917851e6c4d2f898e7388352ab5fc290f2 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Wed, 21 Jul 2021 13:49:41 +0200 Subject: [PATCH 02/24] Added: workflow file --- .clang-format | 34 ++++++++++++++- .github/workflows/lint.yml | 42 +++++++++++++++++++ tools/license_header.py | 3 ++ .../{git-clang-format => git-clang-format-13} | 20 +++------ 4 files changed, 82 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/lint.yml rename tools/lint/{git-clang-format => git-clang-format-13} (98%) mode change 100644 => 100755 diff --git a/.clang-format b/.clang-format index 30aec4f3150e..e1f694f5f1cd 100644 --- a/.clang-format +++ b/.clang-format @@ -19,7 +19,6 @@ Language: Cpp BasedOnStyle: Google ColumnLimit: 100 -#AlignAfterOpenBracket: AlwaysBreak AlignConsecutiveAssignments: true AlignConsecutiveDeclarations: false AlignConsecutiveMacros: true @@ -27,4 +26,35 @@ DerivePointerAlignment: false SortIncludes: true MaxEmptyLinesToKeep: 1 PointerAlignment: Left -#Standard: Cpp11 \ No newline at end of file +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: false +BinPackParameters: false +SortIncludes: true +IncludeBlocks: Regroup +IncludeCategories: + # Headers in <> from specific external libraries. + - Regex: '<((mxnet)|(nnvm)|(dmlc)|(gtest))\/([-\/_A-Za-z0-9])+((\.h)|(\.hpp))>' + Priority: 1 + # Headers in <> with extension. + - Regex: '<([-\/_A-Za-z0-9])+((\.h)|(\.hpp))>' + Priority: 2 + # Headers in <> without extension. + - Regex: '<([-\/_A-Za-z0-9])+>' + Priority: 3 + # Local (./) Headers in "" with extension. + - Regex: '"(\.\/)([-\/_A-Za-z0-9])+((\.h)|(\.hpp))"' + Priority: 4 + # Local (../) Headers in "" with extension. + - Regex: '"(\.\.\/)+([-\/_A-Za-z0-9])+((\.h)|(\.hpp))"' + Priority: 5 + # Headers in "" with extension. + - Regex: '"([-\/_A-Za-z0-9])+((\.h)|(\.hpp))"' + Priority: 6 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000000..cf842938aa29 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,42 @@ +name: Clang format lint + +on: + push: + branches: + - master + pull_request: + +jobs: + clang-format: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Fetch MxNet + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Run clang-format + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -eu + git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git + git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" + + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "| clang-format failures found! Run: " + echo "| tool/lint/clang_format_ci.sh ${BASE_SHA} " + echo "| to fix this error. " + echo "| For more info, see: " + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + + tools/lint/clang_format_ci.sh "${BASE_SHA}" + + GIT_DIFFERENCE=$(git diff) + if [[ -z $GIT_DIFFERENCE ]]; then + git remote remove "${GITHUB_RUN_ID}" # temporary remote is removed + exit 0 + fi + echo "$GIT_DIFFERENCE" + git remote remove "${GITHUB_RUN_ID}" # temporary remote is removed + exit 1 diff --git a/tools/license_header.py b/tools/license_header.py index 5efa7ff803b0..89fd33b2ed93 100755 --- a/tools/license_header.py +++ b/tools/license_header.py @@ -77,6 +77,9 @@ '3rdparty/ps-lite', '3rdparty/tvm', + # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + 'tools/lint/git-clang-format-13' + # 3rdparty headerfiles under different licenses 'include/onednn', diff --git a/tools/lint/git-clang-format b/tools/lint/git-clang-format-13 old mode 100644 new mode 100755 similarity index 98% rename from tools/lint/git-clang-format rename to tools/lint/git-clang-format-13 index 0233ceb3a868..cfcfb9fcbaef --- a/tools/lint/git-clang-format +++ b/tools/lint/git-clang-format-13 @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # #===- git-clang-format - ClangFormat Git Integration ---------*- python -*--===# # @@ -85,7 +85,6 @@ def main(): 'js', # JavaScript 'ts', # TypeScript 'cs', # C Sharp - 'json', # Json ]) p = argparse.ArgumentParser( @@ -138,15 +137,10 @@ def main(): if opts.verbose >= 1: ignored_files = set(changed_lines) filter_by_extension(changed_lines, opts.extensions.lower().split(',')) - # The computed diff outputs absolute paths, so we must cd before accessing - # those files. - cd_to_toplevel() - filter_symlinks(changed_lines) if opts.verbose >= 1: ignored_files.difference_update(changed_lines) if ignored_files: - print( - 'Ignoring changes in the following files (wrong extension or symlink):') + print('Ignoring changes in the following files (wrong extension):') for filename in ignored_files: print(' %s' % filename) if changed_lines: @@ -157,6 +151,9 @@ def main(): if opts.verbose >= 0: print('no modified files to format') return + # The computed diff outputs absolute paths, so we must cd before accessing + # those files. + cd_to_toplevel() if len(commits) > 1: old_tree = commits[1] new_tree = run_clang_format_and_save_to_tree(changed_lines, @@ -340,13 +337,6 @@ def filter_by_extension(dictionary, allowed_extensions): del dictionary[filename] -def filter_symlinks(dictionary): - """Delete every key in `dictionary` that is a symlink.""" - for filename in list(dictionary.keys()): - if os.path.islink(filename): - del dictionary[filename] - - def cd_to_toplevel(): """Change to the top level of the git repository.""" toplevel = run('git', 'rev-parse', '--show-toplevel') From 0f42f1ebfd64dc7733ae8bb420d559b5da44631e Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Wed, 21 Jul 2021 13:52:26 +0200 Subject: [PATCH 03/24] tools/lint/clang_format_ci.sh was added --- tools/lint/clang_format_ci.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tools/lint/clang_format_ci.sh diff --git a/tools/lint/clang_format_ci.sh b/tools/lint/clang_format_ci.sh new file mode 100644 index 000000000000..41823b45e757 --- /dev/null +++ b/tools/lint/clang_format_ci.sh @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#!/bin/sh +set -eux + tools/lint/git-clang-format-13 --verbose "$1" -- From 6889cb82d0ba41532495b5088ace4fa5a21c65e9 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Wed, 21 Jul 2021 13:54:42 +0200 Subject: [PATCH 04/24] Permision was set on +x --- tools/lint/clang_format_ci.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tools/lint/clang_format_ci.sh diff --git a/tools/lint/clang_format_ci.sh b/tools/lint/clang_format_ci.sh old mode 100644 new mode 100755 From 4664143615e46ee4182d4fee3bcd73ccf5cb699f Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Fri, 23 Jul 2021 11:44:11 +0200 Subject: [PATCH 05/24] Jenkins clang-format runner --- .github/workflows/lint.yml | 42 ---------------------------------- ci/docker/runtime_functions.sh | 25 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 42 deletions(-) delete mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index cf842938aa29..000000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Clang format lint - -on: - push: - branches: - - master - pull_request: - -jobs: - clang-format: - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - steps: - - name: Fetch MxNet - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Run clang-format - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - set -eu - git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git - git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" - - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| clang-format failures found! Run: " - echo "| tool/lint/clang_format_ci.sh ${BASE_SHA} " - echo "| to fix this error. " - echo "| For more info, see: " - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - - tools/lint/clang_format_ci.sh "${BASE_SHA}" - - GIT_DIFFERENCE=$(git diff) - if [[ -z $GIT_DIFFERENCE ]]; then - git remote remove "${GITHUB_RUN_ID}" # temporary remote is removed - exit 0 - fi - echo "$GIT_DIFFERENCE" - git remote remove "${GITHUB_RUN_ID}" # temporary remote is removed - exit 1 diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 70934f64080c..780dbe8c9dce 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -701,6 +701,7 @@ build_ubuntu_gpu_large_tensor() { sanity_check() { set -ex + sanity_clang sanity_license sanity_python sanity_cpp @@ -716,6 +717,30 @@ sanity_cpp() { 3rdparty/dmlc-core/scripts/lint.py mxnet cpp include src plugin cpp-package tests --exclude_path src/operator/contrib/ctc_include include/onednn } +sanity_clang() { + set -ex + BASE_SHA=${{ github.event.pull_request.base.sha }} + git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git + git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" + git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git + git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "| clang-format failures found! Run: " + echo "| tool/lint/clang_format_ci.sh ${BASE_SHA} " + echo "| to fix this error. " + echo "| For more info, see: " + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + tools/lint/clang_format_ci.sh "${BASE_SHA}" + GIT_DIFFERENCE=$(git diff) + if [[ -z $GIT_DIFFERENCE ]]; then + git remote remove "${GITHUB_RUN_ID}" # temporary remote is removed + exit 0 + fi + echo "$GIT_DIFFERENCE" + git remote remove "${GITHUB_RUN_ID}" # temporary remote is removed + exit 1 +} + sanity_python() { set -ex export DMLC_LOG_STACK_TRACE_DEPTH=100 From 9f8557a4340c274265d90a6bc8eb20cc980b5435 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Fri, 23 Jul 2021 11:53:40 +0200 Subject: [PATCH 06/24] Update runtime_funciton.sh file --- ci/docker/runtime_functions.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 780dbe8c9dce..c9622636fcd4 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -722,8 +722,6 @@ sanity_clang() { BASE_SHA=${{ github.event.pull_request.base.sha }} git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" - git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git - git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "| clang-format failures found! Run: " echo "| tool/lint/clang_format_ci.sh ${BASE_SHA} " From b0861b5dea9a7b5c43015fbd0158129e21f9d3c7 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Tue, 27 Jul 2021 10:39:57 +0200 Subject: [PATCH 07/24] Master last commit sha --- ci/docker/runtime_functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index c9622636fcd4..461479864b51 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -719,7 +719,7 @@ sanity_cpp() { sanity_clang() { set -ex - BASE_SHA=${{ github.event.pull_request.base.sha }} + BASE_SHA=$(git show-ref --hash refs/heads/master) git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" From 8af334f1687e5253474d49b64385e217be1eb195 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Tue, 27 Jul 2021 10:59:35 +0200 Subject: [PATCH 08/24] Set BASE_SHA in greetings --- .github/workflows/greetings.yml | 2 ++ ci/docker/runtime_functions.sh | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml index a9db175035aa..80f9f86fb0c5 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/greetings.yml @@ -7,6 +7,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/first-interaction@v1 + env: + GITHUB_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} with: repo-token: ${{ secrets.GITHUB_TOKEN }} issue-message: | diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 461479864b51..1ffd7697ee06 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -717,9 +717,11 @@ sanity_cpp() { 3rdparty/dmlc-core/scripts/lint.py mxnet cpp include src plugin cpp-package tests --exclude_path src/operator/contrib/ctc_include include/onednn } +# .github/workgflows/greetings.yml passes BASE_SHA for pull requests. +# That's the place we want to check for PRs. sanity_clang() { set -ex - BASE_SHA=$(git show-ref --hash refs/heads/master) + BASE_SHA="${GITHUB_PR_BASE_SHA}" git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" From 0b9962976bca83dbdfd07bf963f7994d49dc7874 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Tue, 27 Jul 2021 11:20:37 +0200 Subject: [PATCH 09/24] GITHUB_BASE_REF and GITHUB_RUN_ID: set varaibles --- .github/workflows/greetings.yml | 2 ++ ci/docker/runtime_functions.sh | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml index 80f9f86fb0c5..358694a8d7b9 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/greetings.yml @@ -9,6 +9,8 @@ jobs: - uses: actions/first-interaction@v1 env: GITHUB_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + GITHUB_PR_RUN_ID: ${{ GITHUB_RUN_ID }} + GITHUB_PR_BASE_REF: ${{ GITHUB_BASE_REF }} with: repo-token: ${{ secrets.GITHUB_TOKEN }} issue-message: | diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 1ffd7697ee06..7eefde16a2d5 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -717,11 +717,12 @@ sanity_cpp() { 3rdparty/dmlc-core/scripts/lint.py mxnet cpp include src plugin cpp-package tests --exclude_path src/operator/contrib/ctc_include include/onednn } -# .github/workgflows/greetings.yml passes BASE_SHA for pull requests. -# That's the place we want to check for PRs. sanity_clang() { set -ex + # .github/workgflows/greetings.yml passes BASE_SHA for pull requests. BASE_SHA="${GITHUB_PR_BASE_SHA}" + GITHUB_RUN_ID="${GITHUB_PR_RUN_ID}" + GITHUB_BASE_REF="${GITHUB_PR_BASE_REF}" git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" From 6c0667af2efcb4eae6a8741baf41b0434ff16a98 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Tue, 27 Jul 2021 11:43:56 +0200 Subject: [PATCH 10/24] Runtime function, os_x_static_build stores configuration --- .github/workflows/greetings.yml | 4 ---- .github/workflows/os_x_staticbuild.yml | 5 ++++- ci/docker/runtime_functions.sh | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml index 358694a8d7b9..a9db175035aa 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/greetings.yml @@ -7,10 +7,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/first-interaction@v1 - env: - GITHUB_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - GITHUB_PR_RUN_ID: ${{ GITHUB_RUN_ID }} - GITHUB_PR_BASE_REF: ${{ GITHUB_BASE_REF }} with: repo-token: ${{ secrets.GITHUB_TOKEN }} issue-message: | diff --git a/.github/workflows/os_x_staticbuild.yml b/.github/workflows/os_x_staticbuild.yml index 1dd6e9c926d2..fa44a438aae2 100644 --- a/.github/workflows/os_x_staticbuild.yml +++ b/.github/workflows/os_x_staticbuild.yml @@ -8,7 +8,10 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - + env: + GITHUB_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + GITHUB_PR_RUN_ID: ${{ GITHUB_RUN_ID }} + GITHUB_PR_BASE_REF: ${{ GITHUB_BASE_REF }} - name: Compilation cache uses: actions/cache@v2 with: diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 7eefde16a2d5..e3062dda9af2 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -719,7 +719,7 @@ sanity_cpp() { sanity_clang() { set -ex - # .github/workgflows/greetings.yml passes BASE_SHA for pull requests. + # .github/workgflows/os_x_staticbuild.yml passes BASE_SHA, GITHUB_RUN_ID, GITHUB_BASE_REF for pull requests. BASE_SHA="${GITHUB_PR_BASE_SHA}" GITHUB_RUN_ID="${GITHUB_PR_RUN_ID}" GITHUB_BASE_REF="${GITHUB_PR_BASE_REF}" From bcd39e2fcfbdfcd9bb766df69c9dff23f7f3d383 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Tue, 27 Jul 2021 11:47:42 +0200 Subject: [PATCH 11/24] Greetings contains env variables --- .github/workflows/greetings.yml | 4 ++++ .github/workflows/os_x_staticbuild.yml | 5 +---- ci/docker/runtime_functions.sh | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml index a9db175035aa..358694a8d7b9 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/greetings.yml @@ -7,6 +7,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/first-interaction@v1 + env: + GITHUB_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + GITHUB_PR_RUN_ID: ${{ GITHUB_RUN_ID }} + GITHUB_PR_BASE_REF: ${{ GITHUB_BASE_REF }} with: repo-token: ${{ secrets.GITHUB_TOKEN }} issue-message: | diff --git a/.github/workflows/os_x_staticbuild.yml b/.github/workflows/os_x_staticbuild.yml index fa44a438aae2..1dd6e9c926d2 100644 --- a/.github/workflows/os_x_staticbuild.yml +++ b/.github/workflows/os_x_staticbuild.yml @@ -8,10 +8,7 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v2 - env: - GITHUB_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - GITHUB_PR_RUN_ID: ${{ GITHUB_RUN_ID }} - GITHUB_PR_BASE_REF: ${{ GITHUB_BASE_REF }} + - name: Compilation cache uses: actions/cache@v2 with: diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index e3062dda9af2..711006113a66 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -719,7 +719,7 @@ sanity_cpp() { sanity_clang() { set -ex - # .github/workgflows/os_x_staticbuild.yml passes BASE_SHA, GITHUB_RUN_ID, GITHUB_BASE_REF for pull requests. + # .github/workgflows/greetings.yml passes BASE_SHA, GITHUB_RUN_ID, GITHUB_BASE_REF for pull requests. BASE_SHA="${GITHUB_PR_BASE_SHA}" GITHUB_RUN_ID="${GITHUB_PR_RUN_ID}" GITHUB_BASE_REF="${GITHUB_PR_BASE_REF}" From ff89ed38a42f58b377e537c7eb78e7ce9f3a9021 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Tue, 27 Jul 2021 14:31:18 +0200 Subject: [PATCH 12/24] Check env params --- ci/docker/runtime_functions.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 711006113a66..d3f357c72066 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -720,9 +720,19 @@ sanity_cpp() { sanity_clang() { set -ex # .github/workgflows/greetings.yml passes BASE_SHA, GITHUB_RUN_ID, GITHUB_BASE_REF for pull requests. - BASE_SHA="${GITHUB_PR_BASE_SHA}" + BASE_SHA="${GITHUB_PR_BASE_SHA:-refs/heads/master}" GITHUB_RUN_ID="${GITHUB_PR_RUN_ID}" GITHUB_BASE_REF="${GITHUB_PR_BASE_REF}" + if [ "${BASE_SHA}" == "refs/heads/master"]; then + BASE_SHA=`git show-ref ${BASE_SHA}` + fi + if [ "${GITHUB_RUN_ID}" == ""]; then + GITHUB_RUN_ID=`(git rev-parse HEAD)` + fi + if [ "${GITHUB_BASE_REF}" == ""]; then + GITHUB_BASE_REF="master" + fi + git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" From e2740140e9a7ed77070561839bb4fddf7a0c0406 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Tue, 27 Jul 2021 15:14:41 +0200 Subject: [PATCH 13/24] White space between breackets --- ci/docker/runtime_functions.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index d3f357c72066..2e39ce0fe2df 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -723,13 +723,13 @@ sanity_clang() { BASE_SHA="${GITHUB_PR_BASE_SHA:-refs/heads/master}" GITHUB_RUN_ID="${GITHUB_PR_RUN_ID}" GITHUB_BASE_REF="${GITHUB_PR_BASE_REF}" - if [ "${BASE_SHA}" == "refs/heads/master"]; then + if [ "${BASE_SHA}" == "refs/heads/master" ]; then BASE_SHA=`git show-ref ${BASE_SHA}` fi - if [ "${GITHUB_RUN_ID}" == ""]; then + if [ "${GITHUB_RUN_ID}" == "" ]; then GITHUB_RUN_ID=`(git rev-parse HEAD)` fi - if [ "${GITHUB_BASE_REF}" == ""]; then + if [ "${GITHUB_BASE_REF}" == "" ]; then GITHUB_BASE_REF="master" fi From 13a7f05de842d402e08fa796fec74ff3bda3760a Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Tue, 27 Jul 2021 15:49:38 +0200 Subject: [PATCH 14/24] Show all refs --- ci/docker/runtime_functions.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 2e39ce0fe2df..927fd28614d5 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -723,24 +723,25 @@ sanity_clang() { BASE_SHA="${GITHUB_PR_BASE_SHA:-refs/heads/master}" GITHUB_RUN_ID="${GITHUB_PR_RUN_ID}" GITHUB_BASE_REF="${GITHUB_PR_BASE_REF}" + if [ "${BASE_SHA}" == "refs/heads/master" ]; then - BASE_SHA=`git show-ref ${BASE_SHA}` - fi - if [ "${GITHUB_RUN_ID}" == "" ]; then - GITHUB_RUN_ID=`(git rev-parse HEAD)` - fi - if [ "${GITHUB_BASE_REF}" == "" ]; then - GITHUB_BASE_REF="master" + git show-ref + BASE_SHA=`git show-ref --hash refs/heads/master` + if [ "${GITHUB_RUN_ID}" == "" || "${GITHUB_BASE_REF}" == "" ]; then + GITHUB_RUN_ID=`(git rev-parse HEAD)` + GITHUB_BASE_REF="master" + fi fi - git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "| clang-format failures found! Run: " echo "| tool/lint/clang_format_ci.sh ${BASE_SHA} " echo "| to fix this error. " echo "| For more info, see: " echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + tools/lint/clang_format_ci.sh "${BASE_SHA}" GIT_DIFFERENCE=$(git diff) if [[ -z $GIT_DIFFERENCE ]]; then From 758b9ba8f73474e0d97cbf3dc68efd2b3a362883 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Tue, 27 Jul 2021 16:15:48 +0200 Subject: [PATCH 15/24] Correct refs to master --- ci/docker/runtime_functions.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 927fd28614d5..4e347c2434a5 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -720,15 +720,14 @@ sanity_cpp() { sanity_clang() { set -ex # .github/workgflows/greetings.yml passes BASE_SHA, GITHUB_RUN_ID, GITHUB_BASE_REF for pull requests. - BASE_SHA="${GITHUB_PR_BASE_SHA:-refs/heads/master}" + BASE_SHA="${GITHUB_PR_BASE_SHA:-refs/remotes/origin/master}" GITHUB_RUN_ID="${GITHUB_PR_RUN_ID}" GITHUB_BASE_REF="${GITHUB_PR_BASE_REF}" - if [ "${BASE_SHA}" == "refs/heads/master" ]; then - git show-ref - BASE_SHA=`git show-ref --hash refs/heads/master` + if [ "${BASE_SHA}" == "refs/remotes/origin/master" ]; then + BASE_SHA=`git show-ref --hash refs/remotes/origin/master` if [ "${GITHUB_RUN_ID}" == "" || "${GITHUB_BASE_REF}" == "" ]; then - GITHUB_RUN_ID=`(git rev-parse HEAD)` + GITHUB_RUN_ID=`(git log --pretty=format:'%h' -n 1)` GITHUB_BASE_REF="master" fi fi From 977447d46eda994ef486718b1333e2f625ebbdd4 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Tue, 27 Jul 2021 16:37:25 +0200 Subject: [PATCH 16/24] End up if [] --- ci/docker/runtime_functions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4e347c2434a5..cf1a718ec903 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -726,11 +726,12 @@ sanity_clang() { if [ "${BASE_SHA}" == "refs/remotes/origin/master" ]; then BASE_SHA=`git show-ref --hash refs/remotes/origin/master` - if [ "${GITHUB_RUN_ID}" == "" || "${GITHUB_BASE_REF}" == "" ]; then + if [ "${GITHUB_RUN_ID}" == "" ] || [ "${GITHUB_BASE_REF}" == "" ]; then GITHUB_RUN_ID=`(git log --pretty=format:'%h' -n 1)` GITHUB_BASE_REF="master" fi fi + git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" From a97a7959af49d70e1cd8968208f12f20a24e7496 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Tue, 27 Jul 2021 17:09:15 +0200 Subject: [PATCH 17/24] Git clang format rat-excludes --- rat-excludes | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rat-excludes b/rat-excludes index c9e85a520787..6f75545a4420 100644 --- a/rat-excludes +++ b/rat-excludes @@ -71,6 +71,9 @@ DartConfiguration.tcl .*\.egg-info .*\.t +# SPDX-License-Identifier: git-clang-format-13 +git-clang-format-13 + # Files generated by Cython core.cpp symbol.cpp From 87d2d31e1be6de65753e9e56093e7d9df978b33d Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Thu, 29 Jul 2021 09:07:00 +0200 Subject: [PATCH 18/24] LICENCE file was modifed to exclude clang-format-13 file --- LICENSE | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/LICENSE b/LICENSE index 17099072f64a..502d9c904546 100644 --- a/LICENSE +++ b/LICENSE @@ -353,3 +353,10 @@ ======================================================================================= docs/python_docs/themes/mx-theme/mxtheme/static/webfonts/ (Copy of the License available at licenses/OFL1_1) + + ======================================================================================= + SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + ======================================================================================= + + tools/lint/git-clang-format-13 + \ No newline at end of file From 3b06980c02b9c3f815174def28639c86c2371159 Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Thu, 29 Jul 2021 09:30:19 +0200 Subject: [PATCH 19/24] Greetings, update env --- .github/workflows/greetings.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/greetings.yml b/.github/workflows/greetings.yml index 358694a8d7b9..cdf5acc700c6 100644 --- a/.github/workflows/greetings.yml +++ b/.github/workflows/greetings.yml @@ -9,8 +9,8 @@ jobs: - uses: actions/first-interaction@v1 env: GITHUB_PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - GITHUB_PR_RUN_ID: ${{ GITHUB_RUN_ID }} - GITHUB_PR_BASE_REF: ${{ GITHUB_BASE_REF }} + GITHUB_PR_RUN_ID: ${{ github.run_id }} + GITHUB_PR_BASE_REF: ${{ github.event.pull_request.base.ref }} with: repo-token: ${{ secrets.GITHUB_TOKEN }} issue-message: | From 48d31c50955c2a30f22d61107f560b0837c8847c Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Thu, 29 Jul 2021 10:12:09 +0200 Subject: [PATCH 20/24] Remove unnecessary condition --- ci/docker/runtime_functions.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index cf1a718ec903..a4541152457c 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -720,11 +720,11 @@ sanity_cpp() { sanity_clang() { set -ex # .github/workgflows/greetings.yml passes BASE_SHA, GITHUB_RUN_ID, GITHUB_BASE_REF for pull requests. - BASE_SHA="${GITHUB_PR_BASE_SHA:-refs/remotes/origin/master}" + BASE_SHA="${GITHUB_PR_BASE_SHA}" GITHUB_RUN_ID="${GITHUB_PR_RUN_ID}" GITHUB_BASE_REF="${GITHUB_PR_BASE_REF}" - if [ "${BASE_SHA}" == "refs/remotes/origin/master" ]; then + if [ "${BASE_SHA}" == "" ]; then BASE_SHA=`git show-ref --hash refs/remotes/origin/master` if [ "${GITHUB_RUN_ID}" == "" ] || [ "${GITHUB_BASE_REF}" == "" ]; then GITHUB_RUN_ID=`(git log --pretty=format:'%h' -n 1)` From dab49ba88081f82c3ec803e8dbc2886f2e606e45 Mon Sep 17 00:00:00 2001 From: Sheng Zha Date: Sun, 1 Aug 2021 20:46:43 -0400 Subject: [PATCH 21/24] Update LICENSE --- LICENSE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 502d9c904546..3d6a0bf2f95a 100644 --- a/LICENSE +++ b/LICENSE @@ -355,8 +355,8 @@ docs/python_docs/themes/mx-theme/mxtheme/static/webfonts/ (Copy of the License available at licenses/OFL1_1) ======================================================================================= - SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + Apache-2.0 WITH LLVM-exception ======================================================================================= tools/lint/git-clang-format-13 - \ No newline at end of file + From 3c6e531fc7580c84e7cc480ed6e0973b98197f9b Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Fri, 13 Aug 2021 17:21:31 +0200 Subject: [PATCH 22/24] Fix licence checker --- tools/license_header.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/license_header.py b/tools/license_header.py index 89fd33b2ed93..ccd1bde41abb 100755 --- a/tools/license_header.py +++ b/tools/license_header.py @@ -78,7 +78,7 @@ '3rdparty/tvm', # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - 'tools/lint/git-clang-format-13' + 'tools/lint/git-clang-format-13', # 3rdparty headerfiles under different licenses 'include/onednn', From cf14abf8788a04602590562c30d60a1d77811e9a Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Mon, 13 Sep 2021 13:59:05 +0200 Subject: [PATCH 23/24] Clang-format file update --- .clang-format | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/.clang-format b/.clang-format index e1f694f5f1cd..47a12e6bc998 100644 --- a/.clang-format +++ b/.clang-format @@ -37,24 +37,4 @@ AlwaysBreakBeforeMultilineStrings: true AlwaysBreakTemplateDeclarations: true BinPackArguments: false BinPackParameters: false -SortIncludes: true -IncludeBlocks: Regroup -IncludeCategories: - # Headers in <> from specific external libraries. - - Regex: '<((mxnet)|(nnvm)|(dmlc)|(gtest))\/([-\/_A-Za-z0-9])+((\.h)|(\.hpp))>' - Priority: 1 - # Headers in <> with extension. - - Regex: '<([-\/_A-Za-z0-9])+((\.h)|(\.hpp))>' - Priority: 2 - # Headers in <> without extension. - - Regex: '<([-\/_A-Za-z0-9])+>' - Priority: 3 - # Local (./) Headers in "" with extension. - - Regex: '"(\.\/)([-\/_A-Za-z0-9])+((\.h)|(\.hpp))"' - Priority: 4 - # Local (../) Headers in "" with extension. - - Regex: '"(\.\.\/)+([-\/_A-Za-z0-9])+((\.h)|(\.hpp))"' - Priority: 5 - # Headers in "" with extension. - - Regex: '"([-\/_A-Za-z0-9])+((\.h)|(\.hpp))"' - Priority: 6 +SortIncludes: false From 054f98eaf5059d15a331ecad2582ae782f8040bc Mon Sep 17 00:00:00 2001 From: mozga-intel Date: Wed, 15 Sep 2021 14:11:14 +0200 Subject: [PATCH 24/24] Error message is shifted --- ci/docker/runtime_functions.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index a4541152457c..ded206d36a70 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -734,13 +734,6 @@ sanity_clang() { git remote add "${GITHUB_RUN_ID}" https://github.com/apache/incubator-mxnet.git git fetch "${GITHUB_RUN_ID}" "$GITHUB_BASE_REF" - - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| clang-format failures found! Run: " - echo "| tool/lint/clang_format_ci.sh ${BASE_SHA} " - echo "| to fix this error. " - echo "| For more info, see: " - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" tools/lint/clang_format_ci.sh "${BASE_SHA}" GIT_DIFFERENCE=$(git diff) @@ -748,6 +741,14 @@ sanity_clang() { git remote remove "${GITHUB_RUN_ID}" # temporary remote is removed exit 0 fi + + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "| clang-format failures found! Run: " + echo "| tool/lint/clang_format_ci.sh ${BASE_SHA} " + echo "| to fix this error. " + echo "| For more info, see: " + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "$GIT_DIFFERENCE" git remote remove "${GITHUB_RUN_ID}" # temporary remote is removed exit 1