diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml new file mode 100644 index 0000000000..4636b0edee --- /dev/null +++ b/.github/workflows/pull_request.yaml @@ -0,0 +1,66 @@ +# Developer Notes: +# +# This config is for github actions. Before merging your changes of this file, +# it's recommended to create a PR against the ci-test branch to test if it works +# as expected. + +name: pull_request + +on: + # run on each pull request + pull_request: + types: [ synchronize, reopened, labeled ] + branches: + - master + - 'v[0-9]+.*' # release branch + - ci-test # testing branch for github action + +defaults: + run: + shell: bash + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + container: + image: apachepegasus/clang-format-3.9 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - name: clang-format + run: ./scripts/linux/run-clang-format.py --clang-format-executable=clang-format-3.9 -r src include + + test: + name: Test + needs: lint + runs-on: self-hosted + container: + image: apachepegasus/ci-env + env: + CCACHE_DIR: /tmp/ccache/pegasus + CCACHE_MAXSIZE: 10G + volumes: + # Place ccache compilation intermediate results in host memory, that's shared among containers. + - /tmp/ccache/pegasus:/tmp/ccache/pegasus + # Read docs at https://docs.docker.com/storage/tmpfs/ for more details of using tmpfs in docker. + options: --mount type=tmpfs,destination=/tmp/pegasus,tmpfs-size=10737418240 --cap-add=SYS_PTRACE + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 1 + - name: Unpack prebuilt third-parties + if: contains(github.event.pull_request.labels.*.name, 'thirdparty-modified') == false + run: unzip /root/pegasus-thirdparty-output.zip -d ./thirdparty + - name: Rebuild third-parties + if: contains(github.event.pull_request.labels.*.name, 'thirdparty-modified') + working-directory: thirdparty + run: | + mkdir build + cmake -DCMAKE_BUILD_TYPE=Release -B build/ + cmake --build build/ -j $(($(nproc)/2+1)) + - name: Compilation + run: ./run.sh build -c --skip_thirdparty + - name: Unit Testing + run: ./run.sh test --skip_thirdparty diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8b3f55644d..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,32 +0,0 @@ -sudo: required -dist: bionic - -language: cpp - -compiler: - - gcc - -cache: - - ccache - - apt - -before_install: - - sudo apt-get -y install libaio-dev clang-format-3.9 - -before_script: - - cd thirdparty - - wget https://github.com/XiaoMi/pegasus-common/releases/download/deps/pegasus-thirdparty-prebuild-bionic.zip - - unzip pegasus-thirdparty-prebuild-bionic.zip - - rm -f pegasus-thirdparty-prebuild-bionic.zip - - cd .. - - ulimit -c unlimited -S - -script: - - export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib/:$LD_LIBRARY_PATH - - ./run.sh test --skip_thirdparty --check --disable_gperf - -after_script: - - ./run.sh stop_zk - -notifications: - email: false diff --git a/run.sh b/run.sh index 754aca736e..a46555e51b 100755 --- a/run.sh +++ b/run.sh @@ -207,46 +207,6 @@ function run_build() DISABLE_GPERF="$DISABLE_GPERF" $scripts_dir/build.sh } -##################### -## install -##################### -function usage_install() -{ - echo "Options for subcommand 'install':" - echo " -h|--help print the help info" - echo " -d|--install_dir " - echo " specify the install directory," - echo " if not set, then default is './install'" -} -function run_install() -{ - INSTALL_DIR=$DSN_ROOT - if [ ! -d $INSTALL_DIR ]; then - INSTALL_DIR=`pwd`/install - fi - while [[ $# > 0 ]]; do - key="$1" - case $key in - -h|--help) - usage_install - exit 0 - ;; - -d|--install_dir) - INSTALL_DIR="$2" - shift - ;; - *) - echo "ERROR: unknown option \"$key\"" - echo - usage_install - exit 1 - ;; - esac - shift - done - INSTALL_DIR="$INSTALL_DIR" $scripts_dir/install.sh -} - ##################### ## start_zk ##################### @@ -386,9 +346,6 @@ case $cmd in shift ONLY_BUILD=YES run_build $* ;; - install) - shift - run_install $* ;; test) shift ONLY_BUILD=NO diff --git a/scripts/linux/run-clang-format.py b/scripts/linux/run-clang-format.py new file mode 100755 index 0000000000..4c069a944c --- /dev/null +++ b/scripts/linux/run-clang-format.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python + +# MIT License +# +# Copyright (c) 2017 Guillaume Papin +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +"""A wrapper script around clang-format, suitable for linting multiple files +and to use for continuous integration. + +This is an alternative API for the clang-format command line. +It runs over multiple files and directories in parallel. +A diff output is produced and a sensible exit code is returned. + +""" + +from __future__ import print_function, unicode_literals + +import argparse +import codecs +import difflib +import fnmatch +import io +import errno +import multiprocessing +import os +import signal +import subprocess +import sys +import traceback + +from functools import partial + +try: + from subprocess import DEVNULL # py3k +except ImportError: + DEVNULL = open(os.devnull, "wb") + + +DEFAULT_EXTENSIONS = 'c,h,C,H,cpp,hpp,cc,hh,c++,h++,cxx,hxx' +DEFAULT_CLANG_FORMAT_IGNORE = '.clang-format-ignore' + + +class ExitStatus: + SUCCESS = 0 + DIFF = 1 + TROUBLE = 2 + +def excludes_from_file(ignore_file): + excludes = [] + try: + with io.open(ignore_file, 'r', encoding='utf-8') as f: + for line in f: + if line.startswith('#'): + # ignore comments + continue + pattern = line.rstrip() + if not pattern: + # allow empty lines + continue + excludes.append(pattern) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + return excludes; + +def list_files(files, recursive=False, extensions=None, exclude=None): + if extensions is None: + extensions = [] + if exclude is None: + exclude = [] + + out = [] + for file in files: + if recursive and os.path.isdir(file): + for dirpath, dnames, fnames in os.walk(file): + fpaths = [os.path.join(dirpath, fname) for fname in fnames] + for pattern in exclude: + # os.walk() supports trimming down the dnames list + # by modifying it in-place, + # to avoid unnecessary directory listings. + dnames[:] = [ + x for x in dnames + if + not fnmatch.fnmatch(os.path.join(dirpath, x), pattern) + ] + fpaths = [ + x for x in fpaths if not fnmatch.fnmatch(x, pattern) + ] + for f in fpaths: + ext = os.path.splitext(f)[1][1:] + if ext in extensions: + out.append(f) + else: + out.append(file) + return out + + +def make_diff(file, original, reformatted): + return list( + difflib.unified_diff( + original, + reformatted, + fromfile='{}\t(original)'.format(file), + tofile='{}\t(reformatted)'.format(file), + n=3)) + + +class DiffError(Exception): + def __init__(self, message, errs=None): + super(DiffError, self).__init__(message) + self.errs = errs or [] + + +class UnexpectedError(Exception): + def __init__(self, message, exc=None): + super(UnexpectedError, self).__init__(message) + self.formatted_traceback = traceback.format_exc() + self.exc = exc + + +def run_clang_format_diff_wrapper(args, file): + try: + ret = run_clang_format_diff(args, file) + return ret + except DiffError: + raise + except Exception as e: + raise UnexpectedError('{}: {}: {}'.format(file, e.__class__.__name__, + e), e) + + +def run_clang_format_diff(args, file): + try: + with io.open(file, 'r', encoding='utf-8') as f: + original = f.readlines() + except IOError as exc: + raise DiffError(str(exc)) + + if args.in_place: + invocation = [args.clang_format_executable, '-i', file] + else: + invocation = [args.clang_format_executable, file] + + if args.style: + invocation.extend(['--style', args.style]) + + if args.dry_run: + print(" ".join(invocation)) + return [], [] + + # Use of utf-8 to decode the process output. + # + # Hopefully, this is the correct thing to do. + # + # It's done due to the following assumptions (which may be incorrect): + # - clang-format will returns the bytes read from the files as-is, + # without conversion, and it is already assumed that the files use utf-8. + # - if the diagnostics were internationalized, they would use utf-8: + # > Adding Translations to Clang + # > + # > Not possible yet! + # > Diagnostic strings should be written in UTF-8, + # > the client can translate to the relevant code page if needed. + # > Each translation completely replaces the format string + # > for the diagnostic. + # > -- http://clang.llvm.org/docs/InternalsManual.html#internals-diag-translation + # + # It's not pretty, due to Python 2 & 3 compatibility. + encoding_py3 = {} + if sys.version_info[0] >= 3: + encoding_py3['encoding'] = 'utf-8' + + try: + proc = subprocess.Popen( + invocation, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + **encoding_py3) + except OSError as exc: + raise DiffError( + "Command '{}' failed to start: {}".format( + subprocess.list2cmdline(invocation), exc + ) + ) + proc_stdout = proc.stdout + proc_stderr = proc.stderr + if sys.version_info[0] < 3: + # make the pipes compatible with Python 3, + # reading lines should output unicode + encoding = 'utf-8' + proc_stdout = codecs.getreader(encoding)(proc_stdout) + proc_stderr = codecs.getreader(encoding)(proc_stderr) + # hopefully the stderr pipe won't get full and block the process + outs = list(proc_stdout.readlines()) + errs = list(proc_stderr.readlines()) + proc.wait() + if proc.returncode: + raise DiffError( + "Command '{}' returned non-zero exit status {}".format( + subprocess.list2cmdline(invocation), proc.returncode + ), + errs, + ) + if args.in_place: + return [], errs + return make_diff(file, original, outs), errs + + +def bold_red(s): + return '\x1b[1m\x1b[31m' + s + '\x1b[0m' + + +def colorize(diff_lines): + def bold(s): + return '\x1b[1m' + s + '\x1b[0m' + + def cyan(s): + return '\x1b[36m' + s + '\x1b[0m' + + def green(s): + return '\x1b[32m' + s + '\x1b[0m' + + def red(s): + return '\x1b[31m' + s + '\x1b[0m' + + for line in diff_lines: + if line[:4] in ['--- ', '+++ ']: + yield bold(line) + elif line.startswith('@@ '): + yield cyan(line) + elif line.startswith('+'): + yield green(line) + elif line.startswith('-'): + yield red(line) + else: + yield line + + +def print_diff(diff_lines, use_color): + if use_color: + diff_lines = colorize(diff_lines) + if sys.version_info[0] < 3: + sys.stdout.writelines((l.encode('utf-8') for l in diff_lines)) + else: + sys.stdout.writelines(diff_lines) + + +def print_trouble(prog, message, use_colors): + error_text = 'error:' + if use_colors: + error_text = bold_red(error_text) + print("{}: {} {}".format(prog, error_text, message), file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--clang-format-executable', + metavar='EXECUTABLE', + help='path to the clang-format executable', + default='clang-format') + parser.add_argument( + '--extensions', + help='comma separated list of file extensions (default: {})'.format( + DEFAULT_EXTENSIONS), + default=DEFAULT_EXTENSIONS) + parser.add_argument( + '-r', + '--recursive', + action='store_true', + help='run recursively over directories') + parser.add_argument( + '-d', + '--dry-run', + action='store_true', + help='just print the list of files') + parser.add_argument( + '-i', + '--in-place', + action='store_true', + help='format file instead of printing differences') + parser.add_argument('files', metavar='file', nargs='+') + parser.add_argument( + '-q', + '--quiet', + action='store_true', + help="disable output, useful for the exit code") + parser.add_argument( + '-j', + metavar='N', + type=int, + default=0, + help='run N clang-format jobs in parallel' + ' (default number of cpus + 1)') + parser.add_argument( + '--color', + default='auto', + choices=['auto', 'always', 'never'], + help='show colored diff (default: auto)') + parser.add_argument( + '-e', + '--exclude', + metavar='PATTERN', + action='append', + default=[], + help='exclude paths matching the given glob-like pattern(s)' + ' from recursive search') + parser.add_argument( + '--style', + help='formatting style to apply (LLVM, Google, Chromium, Mozilla, WebKit)') + + args = parser.parse_args() + + # use default signal handling, like diff return SIGINT value on ^C + # https://bugs.python.org/issue14229#msg156446 + signal.signal(signal.SIGINT, signal.SIG_DFL) + try: + signal.SIGPIPE + except AttributeError: + # compatibility, SIGPIPE does not exist on Windows + pass + else: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + colored_stdout = False + colored_stderr = False + if args.color == 'always': + colored_stdout = True + colored_stderr = True + elif args.color == 'auto': + colored_stdout = sys.stdout.isatty() + colored_stderr = sys.stderr.isatty() + + version_invocation = [args.clang_format_executable, str("--version")] + try: + subprocess.check_call(version_invocation, stdout=DEVNULL) + except subprocess.CalledProcessError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + return ExitStatus.TROUBLE + except OSError as e: + print_trouble( + parser.prog, + "Command '{}' failed to start: {}".format( + subprocess.list2cmdline(version_invocation), e + ), + use_colors=colored_stderr, + ) + return ExitStatus.TROUBLE + + retcode = ExitStatus.SUCCESS + + excludes = excludes_from_file(DEFAULT_CLANG_FORMAT_IGNORE) + excludes.extend(args.exclude) + + files = list_files( + args.files, + recursive=args.recursive, + exclude=excludes, + extensions=args.extensions.split(',')) + + if not files: + return + + njobs = args.j + if njobs == 0: + njobs = multiprocessing.cpu_count() + 1 + njobs = min(len(files), njobs) + + if njobs == 1: + # execute directly instead of in a pool, + # less overhead, simpler stacktraces + it = (run_clang_format_diff_wrapper(args, file) for file in files) + pool = None + else: + pool = multiprocessing.Pool(njobs) + it = pool.imap_unordered( + partial(run_clang_format_diff_wrapper, args), files) + pool.close() + while True: + try: + outs, errs = next(it) + except StopIteration: + break + except DiffError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + retcode = ExitStatus.TROUBLE + sys.stderr.writelines(e.errs) + except UnexpectedError as e: + print_trouble(parser.prog, str(e), use_colors=colored_stderr) + sys.stderr.write(e.formatted_traceback) + retcode = ExitStatus.TROUBLE + # stop at the first unexpected error, + # something could be very wrong, + # don't process all files unnecessarily + if pool: + pool.terminate() + break + else: + sys.stderr.writelines(errs) + if outs == []: + continue + if not args.quiet: + print_diff(outs, use_color=colored_stdout) + if retcode == ExitStatus.SUCCESS: + retcode = ExitStatus.DIFF + if pool: + pool.join() + return retcode + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/linux/run-clang-format.sh b/scripts/linux/run-clang-format.sh index a0b3248cfc..94f380530b 100755 --- a/scripts/linux/run-clang-format.sh +++ b/scripts/linux/run-clang-format.sh @@ -1,14 +1,8 @@ #!/bin/bash -find src include -name "*.h" -or -name "*.cpp" | xargs clang-format-3.9 -i - -effected_files=$(git status -s) -echo "Checking for files that need clang-format..." -if [ -z "${effected_files}" ]; then - echo "All files are well formatted" - exit 0 -else - echo "Please check if the following files are well formatted:" - echo "${effected_files}" - exit 1 -fi +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +PROJECT_DIR=$(dirname $(dirname "${SCRIPT_DIR}")) +python ${SCRIPT_DIR}/run-clang-format.py \ + --clang-format-executable=clang-format-3.9 \ + -i \ + -r ${PROJECT_DIR}/src ${PROJECT_DIR}/include diff --git a/scripts/linux/start_zk.sh b/scripts/linux/start_zk.sh index 3cc67b8410..dabbbe8436 100755 --- a/scripts/linux/start_zk.sh +++ b/scripts/linux/start_zk.sh @@ -4,6 +4,8 @@ # INSTALL_DIR # PORT +PROJECT_DIR=$(realpath $(dirname $(dirname $(dirname "${BASH_SOURCE[0]}")))) + if [ -z "$INSTALL_DIR" ] then echo "ERROR: no INSTALL_DIR specified" @@ -25,26 +27,24 @@ fi cd $INSTALL_DIR -if [ ! -f zookeeper-3.4.6.tar.gz ]; then - echo "Downloading zookeeper..." - download_url="https://github.com/XiaoMi/pegasus-common/releases/download/deps/zookeeper-3.4.6.tar.gz" - wget -T 5 -t 1 $download_url - if [ $? -ne 0 ]; then - echo "ERROR: download zookeeper failed" - exit 1 - fi +ZOOKEEPER_PKG=${PROJECT_DIR}/thirdparty/build/Download/zookeeper/zookeeper-3.4.10.tar.gz +if [ ! -f ${ZOOKEEPER_PKG} ]; then + echo "no such file \"${ZOOKEEPER_PKG}\"" + echo "please install third-parties first" + exit 1 fi if [ ! -d zookeeper-3.4.6 ]; then echo "Decompressing zookeeper..." - tar xf zookeeper-3.4.6.tar.gz + cp ${ZOOKEEPER_PKG} . + tar xf zookeeper-3.4.10.tar.gz if [ $? -ne 0 ]; then echo "ERROR: decompress zookeeper failed" exit 1 fi fi -ZOOKEEPER_HOME=`pwd`/zookeeper-3.4.6 +ZOOKEEPER_HOME=`pwd`/zookeeper-3.4.10 ZOOKEEPER_PORT=$PORT cp $ZOOKEEPER_HOME/conf/zoo_sample.cfg $ZOOKEEPER_HOME/conf/zoo.cfg diff --git a/src/replica/duplication/test/load_from_private_log_test.cpp b/src/replica/duplication/test/load_from_private_log_test.cpp index 5f3f92ec41..c4547fbea3 100644 --- a/src/replica/duplication/test/load_from_private_log_test.cpp +++ b/src/replica/duplication/test/load_from_private_log_test.cpp @@ -208,69 +208,6 @@ class load_from_private_log_test : public duplication_test_base return mlog; } - void test_restart_duplication2() - { - load_from_private_log load(_replica.get(), duplicator.get()); - - // create a log file indexed 3, starting from 38200 - for (int f = 0; f < 3; f++) { - mutation_log_ptr mlog = create_private_log(); - for (int i = 0; i < 100; i++) { - std::string msg = "hello!"; - mutation_ptr mu = create_test_mutation(38000 + 100 * f + i, msg); - mlog->append(mu, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr, 0); - } - mlog->tracker()->wait_outstanding_tasks(); - } - auto files1 = open_log_file_map(_log_dir); - ASSERT_EQ(files1.size(), 3); - boost::filesystem::remove(files1[1]->path()); - boost::filesystem::remove(files1[2]->path()); - boost::filesystem::rename( - files1[3]->path(), - fmt::format("./log.{}.{}", files1[3]->index(), files1[3]->start_offset())); - - // first log is 39100 - { - for (int f = 0; f < 2; f++) { - mutation_log_ptr mlog = create_private_log(); - for (int i = 0; i < 100; i++) { - std::string msg = "hello!"; - mutation_ptr mu = create_test_mutation(39000 + 100 * f + i, msg); - mlog->append(mu, LPC_AIO_IMMEDIATE_CALLBACK, nullptr, nullptr, 0); - } - mlog->tracker()->wait_outstanding_tasks(); - } - boost::filesystem::remove(files1[1]->path()); - } - - { - // This test simulates the following case: - // the replica has written logs [39100 -> 39199], but after some sort of failure, - // it became learner and copied plogs starting from 38200. - boost::filesystem::rename( - fmt::format("./log.{}.{}", files1[3]->index(), files1[3]->start_offset()), - files1[3]->path()); - } - - // log.2.xxx starts from 39100 - // log.3.xxx starts from 38200 - // all log files are reserved for duplication - mutation_log_ptr mlog = create_private_log(); - auto files = mlog->get_log_file_map(); - ASSERT_EQ(files.size(), 2); - - decree max_gced_decree = mlog->max_gced_decree_no_lock(_replica->get_gpid()); - ASSERT_EQ(max_gced_decree, 38199); - - // new duplication, ensure we can start at log.3.xxx - load._private_log = mlog; - load.set_start_decree(max_gced_decree + 1); - load.find_log_file_to_start(); - ASSERT_TRUE(load._current); - ASSERT_EQ(load._current->index(), 3); - } - std::unique_ptr duplicator; }; @@ -337,8 +274,6 @@ TEST_F(load_from_private_log_test, handle_real_private_log) TEST_F(load_from_private_log_test, restart_duplication) { test_restart_duplication(); } -TEST_F(load_from_private_log_test, restart_duplication2) { test_restart_duplication2(); } - TEST_F(load_from_private_log_test, ignore_useless) { utils::filesystem::remove_path(_log_dir);