buildtool

#!/usr/bin/env python3

import argparse
import contextlib
import logging
import os
import platform
import re
import shutil
import subprocess
import sys

__version__ = "0.2.0"

prog_name = os.path.basename(sys.argv[0])

extra_environ_vars = {}

download_dirs = ["~/download/programming/vim", "/tools/software/vim"]

configure_options = """
    --quiet
    --with-features=huge
    --enable-gui=gtk3
    --enable-perlinterp
    --enable-pythoninterp
    --enable-python3interp
    --enable-tclinterp
    --enable-rubyinterp
    --enable-cscope
    --with-compiledby=vim@drmikehenry.com
    """.split()

ruby_command = "/usr/bin/ruby"
if os.path.exists(ruby_command):
    configure_options.extend(["--with-ruby-command=" + ruby_command])

tmp_dir = "build"

usage = """%(prog)s [-h] [--version] [-v] [-q] CMD
For more detailed help: %(prog)s --help
"""

help_description = """
buildtool aids in the compilation of Vim.
"""

help_epilog = """
For help on specific CMD:    %%(prog)s CMD --help

Typical CMD invocations:
  %%(prog)s build [SOURCE]   - unpack, packagesrc, configure, make, package
  %%(prog)s update           - "git pull" in newest Git repo in download_dirs

Less common CMD invocations:
  %%(prog)s unpack [SOURCE]  - unpack source to vim-x.y.z/
  %%(prog)s packagesrc       - make vim-x.y.z.arch.tar.bz2 (source) in PWD
                               (Note: runs "make clean" before packaging!)
  %%(prog)s configure        - run "./configure [options]" in PWD
  %%(prog)s make             - run "make" in PWD
  %%(prog)s package          - make vim-x.y.z.arch.tar.bz2 (binary) in PWD

SOURCE is a Vim source tarball or a Git repo.  If not provided, %%(prog)s
will search through download_dirs (below) for the newest tarball or Git repo to
use:
  %(download_dirs)s

Configuration is done with these options:
  %(configure_options)s
""" % dict(
    download_dirs="\n  ".join(download_dirs),
    configure_options=" ".join(configure_options),
)


# Add a default handler for logging.
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))

logger = logging.getLogger("buildtool")


class BuildError(Exception):
    pass


# Hold parsed arguments in a namespace that may have member functions and
# that allows for more control over storage of argument values.
# Argument values are stored here as regular attributes that match the
# name given in the add_argument() call.
class Args(argparse.Namespace):
    def __init__(self):
        argparse.Namespace.__init__(self)


@contextlib.contextmanager
def preserve_cwd(new_dir=None):
    old_cwd = os.getcwd()
    try:
        if new_dir:
            os.chdir(new_dir)
        yield
    finally:
        os.chdir(old_cwd)


def rm_rf(path):
    if os.path.isfile(path):
        os.remove(path)
    elif os.path.exists(path):
        shutil.rmtree(path)


def mkdir_p(path):
    if not os.path.exists(path):
        os.makedirs(path)


def path_join(*parts):
    return os.path.join(*(p for p in parts if p))


def get_run_env():
    env = os.environ.copy()
    env.update(extra_environ_vars)
    return env


def run(args):
    logger.debug("  " + " ".join(args))
    output_bytes = subprocess.check_output(
        args, env=get_run_env(), stderr=subprocess.STDOUT
    )
    return output_bytes.decode(errors="ignore")


def run_pipe(args):
    logger.debug("  " + " ".join(args))
    process = subprocess.Popen(
        args,
        env=get_run_env(),
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
    )
    return process


def vim_version_from_version_ch(version_ch):
    """version_ch is text of Vim source tree's version.c + version.h.
    """
    major = -1
    minor = -1
    patch = -1
    in_patches = False
    for line in version_ch.splitlines():
        if in_patches:
            m = re.match(r"^\s*(\d+)\s*,?\s*$", line)
            if m:
                patch = max(patch, int(m.group(1)))
            elif re.match(r"^\s*\}", line):
                in_patches = False
        elif re.search(r"^\s*static\s+int\s+included_patches\[\]\s*=", line):
            in_patches = True

        else:
            m = re.match(
                r"\s*#define\s+VIM_VERSION_(MAJOR|MINOR)\s+(\d+)\s*$", line
            )
            if m:
                value = int(m.group(2))
                if m.group(1) == "MAJOR":
                    major = value
                else:
                    minor = value

    if in_patches or major < 0 or minor < 0 or patch < 0:
        raise BuildError("Could not determine version from Vim source")

    version = "%d.%d.%04d" % (major, minor, patch)
    return version


def vim_version_from_source_tree(tree="."):
    c = path_join(tree, "src/version.c")
    h = path_join(tree, "src/version.h")
    try:
        source = open(c).read() + open(h).read()
    except OSError:
        raise BuildError("not a valid Vim source tree")
    return vim_version_from_version_ch(source)


def could_be_git_repo(path):
    return os.path.exists(path_join(path, ".git")) or os.path.exists(
        path_join(path, "refs")
    )


def vim_version_ch_from_git(git_source):
    if not could_be_git_repo(git_source):
        return ""
    with preserve_cwd(git_source):
        try:
            version_ch = run(
                "git show HEAD:src/version.c HEAD:src/version.h".split()
            )
        except subprocess.CalledProcessError:
            version_ch = ""
    return version_ch


def vim_version_from_git(git_source):
    version_ch = vim_version_ch_from_git(git_source)
    if not version_ch:
        raise BuildError("%s is not a Vim git repository" % git_source)
    return vim_version_from_version_ch(version_ch)


def download_dir():
    for d in download_dirs:
        d = os.path.expanduser(d)
        if os.path.isdir(d):
            return d
    raise BuildError("cannot find download directory")


def versioned_tarballs(root):
    for root, dirs, files in os.walk(download_dir()):
        for f in files:
            m = re.match(r"vim-(\d+\.\d+(\.\d+)?)\.tar\.(gz|bz2)$", f)
            if m:
                yield m.group(1), path_join(root, f)


def versioned_git_repos(root):
    for root, dirs, files in os.walk(download_dir()):
        version_ch = vim_version_ch_from_git(root)
        if version_ch:
            version = vim_version_from_version_ch(version_ch)
            yield version, root
            del dirs[:]


def versioned_sources(root):
    return list(versioned_tarballs(root)) + list(versioned_git_repos(root))


def sorted_versioned_sources(ver_sources):
    return sorted(
        ver_sources,
        key=lambda version_path: list(map(int, version_path[0].split("."))),
        reverse=True,
    )


def guess_source():
    candidates = sorted_versioned_sources(versioned_sources(download_dir()))
    if candidates:
        for version, source in candidates:
            logger.debug("Found Vim version {} at {}".format(version, source))
        version, source = candidates[0]
        print("Using Vim {} from {}".format(version, source))
        return source
    return ""


def resolve_source(source):
    """Resolve source into file (tarball) or directory (git repo)."""
    if not source:
        source = guess_source()
        if not source:
            raise BuildError("could not guess source")

    # Files must be tarballs.
    if os.path.isfile(source):
        return source

    if not os.path.isdir(source):
        raise BuildError("source must be tarball or directory")

    # If source is a Vim git repository, return it.
    if vim_version_ch_from_git(source):
        return source

    raise BuildError("bad source %s" % source)


def verify_empty_unpack_dest(dest):
    if os.path.exists(dest):
        raise BuildError("must remove %s before unpacking" % dest)


def start_phase(phase):
    print()
    print("### " + phase)


def update():
    candidates = sorted_versioned_sources(versioned_git_repos(download_dir()))
    if not candidates:
        raise BuildError("cannot find a Git repo to update")
    version, repo = candidates[0]
    print("Updating Git repo %s" % repo)
    with preserve_cwd(repo):
        run("git pull".split())
    print("Old version = %s" % version)
    print("New version = %s" % vim_version_from_git(repo))


def unpack_tar(source):
    tmp_extract_dir = "__TMP_EXTRACT__"
    if os.path.exists(tmp_extract_dir):
        raise BuildError(
            "must remove %s for tarball extraction" % tmp_extract_dir
        )
    try:
        print("Extracting %s into temp area %s" % (source, tmp_extract_dir))
        os.mkdir(tmp_extract_dir)
        run(["tar", "-C", tmp_extract_dir, "-xf", source])

        extracted = os.listdir(tmp_extract_dir)
        if len(extracted) != 1:
            raise BuildError("invalid tarball %s" % source)
        tar_root = path_join(tmp_extract_dir, extracted[0])
        version = vim_version_from_source_tree(tar_root)
        print("Found version %s" % version)
        dest_dir = "vim-" + version
        verify_empty_unpack_dest(dest_dir)
        os.rename(tar_root, dest_dir)
        return dest_dir

    finally:
        rm_rf(tmp_extract_dir)


def unpack_git(source):
    version = vim_version_from_git(source)
    dest_dir = "vim-" + version
    verify_empty_unpack_dest(dest_dir)
    print("Cloning Git source {} into {}".format(source, dest_dir))
    run(["git", "clone", source, dest_dir])
    return dest_dir


def package_src(dest_dir=None):
    start_phase("packagesrc")
    with preserve_cwd(dest_dir):
        version = vim_version_from_source_tree()
        logger.debug("Packaging source for version %s" % version)

        src_tar_name = "vim-%s.tar.bz2" % version
        rm_rf(src_tar_name)

        # To ensure the proper directory structure, first tar everything into
        # a temporary tarball, extract into a temporary directory with the
        # proper parent name, then create the final tarball.

        print('Performing "make distclean"')
        make_dist_clean()
        mkdir_p(tmp_dir)
        src_dir_name = "vim-%s" % version
        src_dir = path_join(tmp_dir, src_dir_name)
        mkdir_p(src_dir)
        tmp_tar = path_join(tmp_dir, "tmp.tar")
        # Create temporary tarfile with no enclosing parent directory, e.g.:
        #   tar -cf build/tmp.tar \
        #   --exclude .git --exclude *.gz --exclude *.bz2 --exclude build .
        run(
            [
                "tar",
                "-cf",
                tmp_tar,
                "--exclude",
                ".git",
                "--exclude",
                "*.gz",
                "--exclude",
                "*.bz2",
                "--exclude",
                tmp_dir,
                ".",
            ]
        )

        # Extract back into src_dir, e.g.::
        #   tar -C build/vim-8.1.0216 -xf build/tmp.tar
        run(["tar", "-C", src_dir, "-xf", tmp_tar])

        # Create final tarfile with proper parent directory, e.g.::
        #   tar -C build -cf vim-8.1.0216.tar.bz2 vim-8.1.0216
        run(["tar", "-C", tmp_dir, "-cf", src_tar_name, src_dir_name])
        print("Created source archive %s" % path_join(dest_dir, src_tar_name))


def unpack(source):
    start_phase("unpack")
    if os.path.isfile(source):
        dest_dir = unpack_tar(source)
    elif os.path.isdir(source):
        dest_dir = unpack_git(source)
    else:
        raise BuildError("source is neither file nor directory")
    print("Unpacked to %s" % dest_dir)
    return dest_dir


def configure(dest_dir=None):
    start_phase("configure")
    with preserve_cwd(dest_dir):
        print(
            "If configure fails, look at %s"
            % path_join(dest_dir, "src/auto/config.log")
        )
        run(["./configure"] + configure_options)


def make(dest_dir=None):
    start_phase("make")
    with preserve_cwd(dest_dir):
        log_path = path_join(tmp_dir, "build.log")
        print(
            "On build failure, examine %s for errors"
            % path_join(dest_dir, log_path)
        )
        mkdir_p(tmp_dir)

        p = run_pipe(["make"])

        i = 0
        width = 40
        with open(log_path, "wb") as log:
            for line_bytes in iter(p.stdout.readline, b""):
                log.write(line_bytes)

                line = line_bytes.decode(errors="ignore")
                if line.startswith("gcc"):
                    msg = "[compile] %s" % line.split()[-1]
                else:
                    msg = "Working %s" % (r"-\|/"[i])
                    i = (i + 1) % 4
                msg = msg.ljust(width)
                sys.stdout.write("\r" + msg)
                sys.stdout.flush()
        print("\r" + " " * width)
        sys.stdout.flush()
        exit_status = p.wait()
        if exit_status != 0:
            raise BuildError("error running make")


def make_dist_clean(dest_dir=None):
    with preserve_cwd(dest_dir):
        run("make distclean".split())


def package(dest_dir=None):
    start_phase("package")
    with preserve_cwd(dest_dir):
        version = vim_version_from_source_tree()
        arch = platform.uname()[-1]
        logger.debug("Packaging for version {} on {}".format(version, arch))

        bin_tar_name = "vim-{}.{}.tar.gz".format(version, arch)

        tar_opts = []
        try:
            if "GNU tar" in run("tar --version".split()):
                tar_opts = "--owner=0 --group=0".split()
        except subprocess.CalledProcessError:
            pass

        mkdir_p(tmp_dir)
        dest = path_join(tmp_dir, "DESTDIR")
        rm_rf(dest)
        run(["make", "install", "DESTDIR=" + os.path.abspath(dest)])
        run("chmod -R u=rwX,go=rX".split() + [dest])
        run(
            ["tar", "-C", dest, "--numeric-owner"]
            + tar_opts
            + ["-zcf", bin_tar_name, "."]
        )
        tar_path = path_join(dest_dir, bin_tar_name)
        print("Created binary archive %s" % tar_path)
        print("To install:")
        print("sudo tar -C / -xf %s" % tar_path)


def setup_fedora_environment():
    r"""
    Setup environment variables needed on Fedora 23+.

    On Fedora 23+, libraries are built with "hardened" settings.  They are
    automatically applied when building an RPM, but not when compiling by hand.
    These variables are needed (at least through vim-8.0.0133) when compiling
    with --enable-rubyinterp and using Fedora-supplied ruby libraries.  Vim's
    configure script asks the ruby interpreter what LDFLAGS to use without also
    asking about CFLAGS; this leads to mismatched values of certain "hardened"
    settings that are now the default in all RPM builds.  The %configure macro
    in RPM SPEC files sets up both CFLAGS and LDFLAGS to their %hardened
    values, which is why Vim's configure script works when building an %RPM,
    but these settings must be applied manually when not building an RPM.

    The variables may be extracted from ``rpmbuild`` via::

      eval $(rpmbuild --eval '%{configure}' | egrep '^\s*[A-Z]+=')

    Example variables are::

      CFLAGS="${CFLAGS:--O2 -g -pipe -Wall -Werror=format-security
        -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions
        -fstack-protector-strong -grecord-gcc-switches
        -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1
        -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic
        -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection}"

      CXXFLAGS="${CXXFLAGS:--O2 -g -pipe -Wall -Werror=format-security
        -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions
        -fstack-protector-strong -grecord-gcc-switches
        -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1
        -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic
        -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection}"

      FFLAGS="${FFLAGS:--O2 -g -pipe -Wall -Werror=format-security
        -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions
        -fstack-protector-strong -grecord-gcc-switches
        -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1
        -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic
        -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection
        -I/usr/lib64/gfortran/modules}"

      FCFLAGS="${FCFLAGS:--O2 -g -pipe -Wall -Werror=format-security
        -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions
        -fstack-protector-strong -grecord-gcc-switches
        -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1
        -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic
        -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection
        -I/usr/lib64/gfortran/modules}"

      LDFLAGS="${LDFLAGS:--Wl,-z,relro  -Wl,-z,now
        -specs=/usr/lib/rpm/redhat/redhat-hardened-ld}"

    """
    if not os.path.exists("/etc/redhat-release"):
        return
    try:
        output = run("rpmbuild --eval %{configure}".split())
    except OSError:
        print(
            "## Warning: could not run rpmbuild; "
            "not setting environment variables"
        )
        return

    env_vars = re.findall(
        r"""
            ^\s*
            ([A-Z]+)
            =
            " \$ \{ \w+ :-
            ([^}"]+)
            }"
        """,
        output,
        re.MULTILINE | re.VERBOSE,
    )
    print("## Detected rpmbuild; adding environment variables")
    for var, value in env_vars:
        logger.debug('{}="{}"'.format(var, value))
        extra_environ_vars[var] = value


def inner_main():
    parser = argparse.ArgumentParser(
        usage=usage,
        description=help_description,
        epilog=help_epilog,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )

    parser.add_argument("--version", action="version", version=__version__)
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_const",
        dest="verbose",
        const=logging.DEBUG,
        help="""verbose output for debugging""",
    )
    parser.add_argument(
        "-q",
        "--quiet",
        action="store_const",
        dest="verbose",
        const=logging.WARNING,
        help="""suppress informational output""",
    )

    subparsers = parser.add_subparsers(dest="cmd", help=argparse.SUPPRESS)

    # 'update' command:
    subparsers.add_parser(
        "update", help='"git pull" in newest Git repo in download_dirs'
    )

    # 'build' command:
    parser_build = subparsers.add_parser(
        "build", help="unpack, packagesrc, configure, build, and package"
    )
    parser_build.add_argument(
        "source", nargs="?", help="Vim source tarball or Git repo"
    )

    # 'unpack' command:
    parser_unpack = subparsers.add_parser(
        "unpack", help="unpack source to vim-x.y.z/"
    )
    parser_unpack.add_argument(
        "source", nargs="?", help="Vim source tarball or Git repo"
    )

    # 'packagesrc' command:
    subparsers.add_parser(
        "packagesrc",
        help="create source package vim-x.y.z.tar.bz2 in PWD (Note: "
        'runs "make clean" before packaging!)',
    )

    # 'configure' command:
    subparsers.add_parser(
        "configure", help='run "./configure [options]" in PWD'
    )

    # 'make' command:
    subparsers.add_parser("make", help='run "make" in PWD')

    # 'package' command:
    subparsers.add_parser(
        "package", help="create binary package vim-x.y.z.arch.tar.bz2 in PWD"
    )

    global args
    args = parser.parse_args(namespace=Args())

    if args.verbose is None:
        logging.getLogger().setLevel(logging.INFO)
    else:
        logging.getLogger().setLevel(args.verbose)

    setup_fedora_environment()

    if args.cmd == "update":
        update()

    elif args.cmd == "build":
        dest_dir = unpack(resolve_source(args.source))
        package_src(dest_dir)
        configure(dest_dir)
        make(dest_dir)
        package(dest_dir)

    elif args.cmd == "unpack":
        unpack(resolve_source(args.source))

    elif args.cmd == "configure":
        configure()

    elif args.cmd == "make":
        make()

    elif args.cmd == "package":
        package()

    elif args.cmd == "packagesrc":
        package_src()


def main():
    try:
        inner_main()
    except BuildError as e:
        print("{}: {}".format(prog_name, e))
        sys.exit(2)


if __name__ == "__main__":
    main()