spackle

#!/bin/bash
#
# Spackle - a local package manager
#
# Install date: @INSTALL_DATE@
# Git commit: @GIT_COMMIT@
#

# Show line num and func name when debugging with 'set -x'
export PS4='+${BASH_SOURCE}:${LINENO}: ' #${FUNCNAME[0]}: '

VERSION="0.1alpha"

# Spackle variables
arch="auto"
prefix="$HOME/local"

# Don't install these packages (ignore them when required as deps)
noInstall=(glibc pacman bash sh awk  perl gcc-libs)

# PACKAGE SOURCES
repos=(abs aur)

declare -A abs
abs[areas]="core extra community"
abs[index]="http://mirrors.kernel.org/archlinux"
abs[server]="rsync.archlinux.org"

declare -A aur
aur[server]="http://aur.archlinux.org/packages"

if [[ "$arch" == "auto" ]]
then
    arch=$(uname -m)
fi

dbDir="$prefix/var/lib/spackle/db"
indexDir="$prefix/var/lib/spackle/index"
packageDir=$(source "$prefix/etc/makepkg.conf"; echo "$PKGDEST")
packageDir=${packageDir:-$prefix/var/lib/spackle/pkg}
buildDir="$prefix/var/lib/spackle/build"
#gpgDir="$prefix/etc/spackle.d/gnupg"
tmpDir="$prefix/tmp"
logFile="$prefix/var/log/spackle.log"

MAKEPKG="$prefix/usr/bin/makepkg --config $prefix/etc/makepkg.conf -dd"
# For downloading abs build files (PKGBUILD and patchfiles)


[[ -e "$prefix/etc/spackle/colors" ]] && source "$prefix/etc/spackle/colors"

success() {
    echo -en "$GREEN" >&2
    echo "$@" >&2
    echo -en "$RESET" >&2
}
emph() {
    echo -en "$LIGHTBLUE" >&2
    echo "$@" >&2
    echo -en "$RESET" >&2
}
report() {
    echo -en "$LIGHTWHITE" >&2
    echo "$@" >&2
    echo -en "$RESET" >&2
}
msg() {
    echo -en "$WHITE" >&2
    echo "$@" >&2
    echo -en "$RESET" >&2
}
warn() {
    echo -en "$YELLOW" >&2
    echo "$@" >&2
    echo -en "$RESET" >&2
}
err() {
    echo -en "$RED" >&2
    echo "$@" >&2
    echo -en "$RESET" >&2
    exit 1
}

abspath() {
    echo "$(cd $(dirname "$1"); pwd -P)/$(basename "$1")"
}


update_aur_index() {
    [[ -e "$indexDir" ]] || mkdir -p "$indexDir"
    # Update from file list of the aur git mirror (approx 6 sec)
    # Updated every hour
    # Kind of a hack, but it was the best index of the entire AUR
    # Alternative: aur3.org has a tarball of all pkgbuilds, but it is
    # only updated nightly, is slower to download, and I don't think
    # aur3.org properly handles special characters (like plus signs)
    # in package names.
    report "Updating ArchLinux AUR index" 
    curl "http://pkgbuild.com/git/aur-mirror.git/plain" | grep '^  <li>' | sed -e 's/<[^>]*>//g' -e 's/^\s*/aur\//' > "$indexDir/aur_index.txt"
}

update_abs_index() {
    [[ -e "$indexDir" ]] || mkdir -p "$indexDir"
    report "Updating ArchLinux ABS index"
    local mirror="${abs[index]}"
    local repo
    for repo in core extra community
    do
        # Rsync lists filenames but not version numbers
        # Also have to get the arch=any packages too
        #rsync "rsync.archlinux.org::abs/x86_64/$repo/" | awk "{print \"$repo/\"\$5}" > "${repo}_index.txt"

        curl -o "$indexDir/$repo.db" "$mirror/$repo/os/$arch/$repo.db"
        tar --exclude="*/*" -tf "$indexDir/$repo.db" | sed -e 's@/$@@' -e "s@^@$repo/@" > "$indexDir/${repo}_index.txt"
    done
}

update_index() {
    update_abs_index
    update_aur_index
}

# Prints a string of the repo, package name, and version
# Name can be specified using a Perl regular expression
#
# Example:
#     find_pkg "pacman"   # core/pacman-4.0.3-1"
#
find_pkg() {
    while [[ "${1:0:1}" == "-" ]]; do
        case "$1" in
        -p|--partial)  local partial_match=true ;;
        *) warn "Skipping unrecognized option '$1'" ;;
        esac
        shift
    done
    local query="$1"
    local found=""
    if [[ "$partial_match" == "true" ]]
    then
        query=".*$query.*"
    fi
    for repo in core extra community
    do
        [[ -e "$indexDir/${repo}_index.txt" ]] || update_abs_index
        grep "/$query-[^-]*-[0-9]*\$" "$indexDir/${repo}_index.txt"
        [[ "$?" == "0" ]] && found="true"
    done

    [[ -e "$indexDir/aur_index.txt" ]] || update_aur_index
    grep "/$query\$" "$indexDir/aur_index.txt"
    [[ "$?" == "0" ]] && found="true"

    [[ "$found" == "true" ]] && return 0 || return 1
}

find_single_pkg() {
    # return first result
    find_pkg "$1" | sed '1 q'
}

urlencode() {
    # if perl and perl's URI module is installed
    perl -n -MURI::Escape -e 'chomp; print uri_escape($_);' 2>/dev/null ||
    # if ruby installed
    ruby -n -r cgi -e 'puts CGI.escape $_.strip' 2>/dev/null ||
    # if python is installed
    python -c "import urllib, sys; print urllib.quote(sys.stdin.readline().strip())" ||
    warn "No urlencode tool found"
}

get_aur3_info() {
    local name="$1"

    curl -Ss "http://aur3.org/rpc/$(echo "$name"|urlencode)"
}

# Prints deps (stripped of version requirements), one dep per line
get_deps_from_aur3() {
    get_aur3_info "$1" | sed -ne '/^ "depends":/,/^ \],/p' | sed -re '1d;$d' -e 's/^ *"//; s/[>].*//; s/",? *$//'
}

get_desc() {
    local entry="$1"
    [[ -z "$entry" ]] && warn "blank package name" && return 1
    [[ "$entry" =~ "/" ]] || entry=$(find_single_pkg "$1")
    [[ "${#entry}" == 0 ]] && warn "package '$1' not found" && return 1
    local repo=$(dirname "$entry")
    local fullname=$(basename "$entry")
    local name=${fullname%-*-*}
    if [[ "$repo" == "aur" ]]
    then
        #curl "http://aur3.org/rpc/$name"
        curl -sS --get -d "type=info" --data-urlencode "arg=$name" "http://aur.archlinux.org/rpc.php" | jshon
    else
        tar -Oxf "$indexDir/$repo.db" "$fullname/desc"
    fi
}

get_property(){
    local entry="$1"
    [[ -z "$entry" ]] && warn "blank package name" && return 1
    [[ "$entry" =~ "/" ]] || entry=$(find_single_pkg "$1")
    [[ "${#entry}" == 0 ]] && warn "package '$1' not found" && return 1
    local property="$2"
    local repo=$(dirname "$entry")
    local fullname=$(basename "$entry")

    if [[ "$repo" == "aur" ]]
    then
        warn "No desc file for AUR packages ($entry)"
        return 1
    fi

    tar -Oxf "$indexDir/$repo.db" "$fullname/desc" |
        sed -n -e "/^%${property^^}%$/,/^$/ {//d; p}"
}

get_pkgbuild() {
    # print PKGBUILD to stdout
    #TODO: don't call find_pkg here
    local entry="$1"
    [[ -z "$entry" ]] && warn "blank package name" && return 1
    [[ "$entry" =~ "/" ]] || entry=$(find_single_pkg "$1")
    [[ "${#entry}" == 0 ]] && warn "package '$1' not found" && return 1
    local repo=$(dirname "$entry")
    local fullname=$(basename "$entry")
    local name=${fullname%-*-*}

    if [[ "$repo" == "aur" ]]
    then
        curl -sS https://aur.archlinux.org/packages/${name:0:2}/$name/PKGBUILD
    else
        local arch=$(get_property "$entry" "arch")
        local base=$(get_property "$entry" "base")
        name="${base:-$name}"
        mkdir -p "$tmpDir/$name"
        rsync "rsync.archlinux.org::abs/$arch/$repo/$name/PKGBUILD" "$tmpDir/$name/PKGBUILD" 
        cat "$tmpDir/$name/PKGBUILD"
        rm -rf "$tmpDir/$name"
    fi
}

get_local_version() {
    local name="$1"
    [[ ! -e "$dbDir/$name/desc" ]] && warn "package '$name' not installed" && return 1
    sed -ne '/%PKGVER%/,/^$/{//d;p}' "$dbDir/$name/desc"
}

get_server_version() {
    # print depends list, one per line
    local entry="$1"
    [[ -z "$entry" ]] && warn "blank package name" && return 1
    [[ "$entry" =~ "/" ]] || entry=$(find_single_pkg "$1")
    [[ "${#entry}" == 0 ]] && warn "package '$1' not found" && return 1
    local repo=${entry%/*}
    local fullname=${entry#*/}
    local name=${fullname%-*-*}
    if [[ "$repo" == "aur" ]]
    then
        curl -sS --get -d "type=info" --data-urlencode "arg=$name" "http://aur.archlinux.org/rpc.php" | jshon -e results -e Version -u
    else
        tar -Oxf "$indexDir/$repo.db" "$fullname/desc" | \
            sed -ne '/^%VERSION%/,/^$/{ //d;  p }' 
    fi
}

get_deps() {
    # print depends list, one per line
    local entry=$(find_single_pkg "$1")
    [[ -z "$entry" ]] && warn "blank package name" && return 1
    [[ "${#entry}" == 0 ]] && warn "package '$1' not found" && return 1
    local repo=${entry%/*}
    local fullname=${entry#*/}
    local name=${fullname%-*-*}
    if [[ "$repo" == "aur" ]]
    then
        # Try to protect against malicious PKGBUILDs by executing them
        # in an empty directory with a restricted bash shell and empty
        # environment. This isn't perfect, but its better than nothing.
        mkdir -p "$tmpDir/empty"
        cd "$tmpDir/empty"
        (
            curl -sS https://aur.archlinux.org/packages/${name:0:2}/$name/PKGBUILD
            function print_deps() {
                IFS=$'\n'
                (( ${#depends[*]} > 0 )) && echo "${depends[*]}"
            }
            declare -f print_deps
            echo print_deps
        #FIXME: set a real path for the rbashrc file
        ) | env - "BASH_ENV=$prefix/etc/spackle/rbashrc" rbash --noprofile --norc
    else
        tar -Oxf "$indexDir/$repo.db" "$fullname/depends" | \
            sed -ne '/^%DEPENDS%/,/^$/{ //d;  p }' 
    fi
}

strip_version() {
    sed -e 's/[<=>].*$//'
}

# TODO: add version checking for dependencies. Use `vercmp` from pacman
# TODO: check for circular dependencies?
get_all_deps() {
    # Bash array quickref
    # create array:   a=(one two three)
    # copy array:     b=("${a[@]}")
    # array append:   a+=("${b[@]}")
    # array pop:      x=$a; a=("${a[@]:1}")
    declare -a ignore=( ${noInstall[@]} )
    case "$1" in 
        --ignore-installed) shift; ignore+=( $(ls -1 "$dbDir") ) ;;
    esac
    local pkg="$1"
    declare -a check=( "$pkg" )
    declare -a deps=()
    ignore+=( "$pkg" )
    local IFS=$'\n'  # join array elements with newline
    while (( ${#check[@]} > 0 ))
    do
        # pop pkg
        local name="${check[0]}"
        check=( "${check[@]:1}" )
        # remove empty string from $check
        (( ${#check[0]} == 0 )) && check=()

        # get deps (filtered by ignore list)
        newdeps="$(get_deps "$name" | strip_version | grep -v -x -F "${ignore[*]}")"
        if (( ${#newdeps} > 0 ))
        then
            check+=( $newdeps )
            ignore+=( $newdeps )
        fi

        [[ "$pkg" != "$name" ]] && deps+=( "$name" )
    done
    echo "${deps[@]}"
}

download_source() {
    local entry="$1"
    [[ -z "$entry" ]] && warn "blank package name" && return 1
    [[ "$entry" =~ "/" ]] || entry=$(find_single_pkg "$1")
    [[ "${#entry}" == 0 ]] && warn "package '$1' not found" && return 1
    local repo=$(dirname "$entry")
    local fullname=$(basename "$entry")
    local name=${fullname%-*-*}

    report "Downloading $entry to '$buildDir/$name'"
    cd "$buildDir"
    rm -rf "$name"

    case "$repo" in
        core|extra|community)
            local arch=$(get_property "$entry" "arch")
            local base=$(get_property "$entry" "base")
            base="${base:-$name}"
            local rsync_flags='-mrtv --no-motd --delete-after --no-p --no-o --no-g'
            rsync -q $rsync_flags "${abs[server]}::abs/$arch/$repo/$base" . 
            ;;
        aur)
            curl "http://aur.archlinux.org/packages/${name:0:2}/$name/$name.tar.gz" | tar -zx
            ;;
        *) warn "Unrecognized repo: '$repo'"
    esac

    cd "$base"
    # install into $prefix
    sed -i -e 's|/usr|$prefix/usr|g' PKGBUILD
    sed -i -e 's|/etc|$prefix/etc|g' PKGBUILD
    sed -i -e 's|/var|$prefix/var|g' PKGBUILD
    sed -i -e 's|/home|$prefix/home|g' PKGBUILD

    # speed up source downloads (ftp.archlinux.org is restricted to 50kb/s)
    sed -i -e 's|ftp://ftp.archlinux.org|http://mirrors.kernel.org/archlinux|g' PKGBUILD

#    if [[ "$name" == "pacman" ]]
#    then
#        patch_pacman_conf
#        patch_makepkg_conf
#        $MAKEPKG --nobuild --skipchecksums
#        (cd src/${fullname%-*}/src/pacman; patch_pacman_c)
#        makepkg_flags="$makepkg_flags --nodeps --skipchecksums --noextract"
#    fi
}

install_package() {
    local no_download build_only dep_for found_source found_package
    local makepkg_flags="--nocheck"
    local currentDir="$PWD"
    while [[ "${1:0:1}" == "-" ]]; do
        case "$1" in
        --build-only)  local build_only=true ;;
        --no-download) local no_download=true ;;
        --dep-for=*)   local dep_for="${1#*=}" ;;
        # makepkg flags
        -d|-dd|--nodeps|\
        --nobuild|\
        --nocheck|\
        --noextract|\
        -f|--force|\
        --skipchecksums|\
        --source)
                       makepkg_flags="$makepkg_flags $1" ;;
        *)             warn "Skipping unrecognized option '$1'" ;;
        esac
        shift
    done

    [[ -e "$buildDir" ]] || mkdir -p "$buildDir"
    [[ -e "$packageDir" ]] || mkdir -p "$packageDir"

    for arg in "$@" 
    do
        local entry=$(find_single_pkg "$arg")
        local repo=$(dirname "$entry")
        local fullname=$(basename "$entry")
        local name=${fullname%-*-*}
        local packages=()

        if [[ -f "$arg" ]] && tar -tf "$arg" ".PKGINFO" &>/dev/null
        then
            # TODO: implement installing from pkg.tar.xz file
            # spackle install foobar.okg.tar.xz
            report "Package found at '$arg'"
            packages+=("$arg")
        elif [[ -z "$entry" ]]
        then
            warn "No package found for '$arg'"
            continue  # skip installation
        elif [[ -e "$packageDir/$fullname-$arch.pkg.tar.xz" ]]
        then
            report "Package found at '$packageDir/$fullname-$arch.pkg.tar.xz'"
            packages+=("$packageDir/$fullname-$arch.pkg.tar.xz")
        elif [[ -e "$packageDir/$fullname-any.pkg.tar.xz" ]]
        then
            report "Package found at '$packageDir/$fullname-any.pkg.tar.xz'"
            packages+=("$packageDir/$fullname-any.pkg.tar.xz")
        else
            local base=$(get_property "$entry" "base")
            if [[ -e "$buildDir/${base:-$name}" ]]
            then
                msg "Build directory '$buildDir/${base:-name}' exists, skip downloading"
            elif [[ "$no_download" != true ]] 
            then
                download_source "$entry"
            fi

            # Check for split package
            # Assume if base is defined, we have a split package
            [[ -n "$base" ]] && [[ "$base" != "$name" ]] && makepkg_flags+=" --pkg $name"

            cd "$buildDir/${base:-$name}"
            report "Building $entry"
            $MAKEPKG $makepkg_flags
            packages+=( $(find "$packageDir" -name "$fullname*.pkg.tar.*") )
            cd "$currentDir"
        fi

        [[ "$build_only" == "true" ]] && continue

        [[ -e "$dbDir" ]] || mkdir -p "$dbDir"

        for package_file in "${packages[@]}"
        do
            report "Installing $(basename $package_file)"
            mkdir -p "$dbDir/$name"
            tar --exclude=".*" -tf "$package_file" > "$dbDir/$name/files"
            tar --exclude=".*" -C "$prefix" -xf "$package_file"
            tar -Oxf "$package_file" ".PKGINFO" | parse_pkginfo > "$dbDir/$name/desc"
            [[ -n "$dep_for" ]] && echo "$dep_for" >> "$dbDir/$name/dep-for"
        done
        (( ${#packages[@]} == 0 )) && err "no packages to install"
    done

    # Re-index
    hash -r
}

upgrade_package() {
    local name choice
    local upgradable=()
    for name in "$@" $([[ "$#" = 0 ]] && ls -1 "$dbDir")
    do
        local local_ver=$(get_local_version "$name")
        [[ -z "$local_ver" ]] && continue
        local remote_ver=$(get_server_version "$name")
        if [[ -z "$remote_ver" ]]
        then
            warn "no remote version found" >&2
            continue
        elif [[ "$local_ver" = "$remote_ver" ]]
        then
            #echo "$name $local_ver (up to date)"
            continue
        fi
        echo "$name $local_ver (upgrade to $remote_ver)"
        upgradable+=($name)
    done
    if (( "${#upgradable[@]}" == 0 ))
    then
        report "System up to date"
        return 0
    fi
    read -p "Upgrade ${#upgradable[@]} package${upgradable[1]:+s}? [y,n] " choice 
    case ${choice:0:1} in  
        y|Y) info "";;
        *) return 1;;
    esac
    for name in "${upgradable[@]}"
    do
        report "Upgrade '$name'"
        uninstall_package $name
        install_package $name
    done
}

# Converts the .PKGINFO in a *.pkg.tar.xz into a desc file
# Usage
# cat .PKGINFO | parse_pkginfo > desc
parse_pkginfo() {
    local key value prev_key add_install_date 
    prev_key=""
    while read line
    do
        [[ "${line:0:1}" == "#" ]] && continue
        # Split on " = "
        key="${line%% = *}"
        value="${line#* = }"
        if [[ "$key" != "$prev_key" ]]
        then
            [[ -n "$prev_key" ]] && echo ""
            # Insert install date after the build date
            # Note: we can convert the posix time to a normal time
            # string with $(date -d "@1339629663")
            if [[ "$prev_key" == "builddate" ]]
            then
                add_install_date=true
                echo "%INSTALLDATE%"
                date '+%s'
                echo ""
            fi
            # Convert key name to upper case
            echo "%${key^^}%"
            prev_key="$key"
        fi
        echo "$value"
    done
    if [[ -z "$add_install_date" ]]
    then
        echo ""
        echo "%INSTALLDATE%"
        date '+%s'
    fi
}

uninstall_package() {
    local name file dir dir_list
    name="$1"
    [[ ! -e "$dbDir/$name" ]] && err "Package '$name' not installed"
    [[ ! -e "$dbDir/$name/files" ]] && err "Package '$name' missing files list"

    # Delete files first
    while read file
    do
        if [[ -d "$prefix/$file" ]]
        then
            dir_list="$file $dir_list"
        else
            rm "$prefix/$file" 2>/dev/null && echo "deleted '$prefix/$file'" >&2
        fi
    done < "$dbDir/$name/files"

    # Delete empty directories
    for dir in $dir_list
    do
        rmdir "$prefix/$dir" 2>/dev/null && echo "deleted '$prefix/$dir'" >&2
    done

    rm -rf "$dbDir/$name"
    report "Uninstalled '$name'"

    hash -r
}

# Convert desc file format to be easily greppable
# Usage:
# cat desc | preparse_desc | grep "^PKGNAME="
# 
# Input:
# %PKGNAME%
# foobar
#
# %PKGVER%
# 1.2.3-4
#
# Output:
# PKGNAME=foobar
# PKGVER=1.2.3-4
preparse_desc() {
    sed -n -e '/^%/{s/%//g; h; d}' \
       -e '/^$/!H' \
       -e '/^$/{x; s/\n/=/; s/\n/\t/g; p}'
}

get_installed() {
    local dir package
    for dir in $(ls -1 "$dbDir")
    do
        cat "$dbDir/$dir/desc" | grep -x -A 1 -E "^%PKGNAME%|^%PKGVER%" |
            grep -v -E "^%|^--$" | tr $'\n' " "
        echo ""
    done
}

get_name_ver_rel() {
    grep -E "^pkgname=|^pkgver=|^pkgrel=" PKGBUILD | awk -F= '{a[$1]=$2}END{OFS="-"; print a["pkgname"],a["pkgver"],a["pkgrel"]}'
}

build_package() {
    install_package --build-only "$@"
}


usage() {
    local script=$(basename "$0")
    echo ""
    echo "$script - a local package manager"
    echo "Commands:"
    echo ""
    echo "    -h          displays help"
    echo "    --help      displays help"
    echo "    --version   displays version"
    echo "    aur3        print package info from aur3.org"
    echo "    build       download and build a package"
    echo "    download    download build files (PKGBUILD & patches)"
    echo "    find        find package_name in search index (Perl regex supported)"
    echo "    find --partial  allow partial name matches"
    echo "    findall     allow partial name matches"
    echo "    info        print package info"
    echo "    install     download, build, and install a package"
    echo "    install --no-download   build and install"
    echo "    install --force         pass '-f' to makepkg"
    echo "    list        list installed packages"
    echo "    pkgbuild    print PKGBUILD for a package"
    echo "    search      find package_name in search index (Perl regex supported)"
    echo "    show        print package info"
    echo "    uninstall   remove an installed package"
    echo "    update      updates the package indices"
    echo "    upgrade     install newer package version"
    echo ""
    echo ""
    echo "Examples:"
    echo ""
    echo "$script update"
    echo "    update the list of packages available from the server"
    echo "$script search zsh"
    echo "    find package with exact name 'zsh'"
    echo "    (repos core, extra, community, and aur are searched)"
    echo "$script search \"pac.*\""
    echo "    find packages that start with 'pac'"
    echo "    (protect regex from bash file expansion by using quotes)"
    echo "$script install zlib"
    echo "    build and installs zlib"
    echo "$script install --force zlib"
    echo "    force makepkg to rebuild zlib package"
    echo "$script install --no-download zlib"
    echo "    builds and installs zlib"
    echo "    (assumes zlib has already been downloaded)"
    echo ""
    echo "$script $VERSION"
    echo ""
}

print_version() {
    local script=$(basename "$0")
    echo "$script $VERSION"
}

case "$1" in
    -h|help|--help)     usage ;;
    --version|version)  print_version ;;
    findall)            shift; find_pkg --partial "$@" ;;
    find|search)        shift; find_pkg "$@" ;;
    update)             update_index ;;
    build)              shift; build_package "$@" ;;
    install)            shift; install_package "$@" ;;
    uninstall)          shift; uninstall_package "$@" ;;
    list)               get_installed ;;
    info|show)          shift; get_desc $(find_single_pkg "$1") ;;
    deps)               shift; get_deps "$1" ;;
    alldeps)            shift; get_all_deps --ignore-installed "$1" ;;
    desc)               shift; get_desc $(find_single_pkg "$1") ;;
    pkgbuild)           shift; get_pkgbuild $(find_single_pkg "$1") ;;
    download)           shift; download_source $(find_single_pkg "$1") ;;
    aur3)               shift; get_aur3_info "$1" ;;
    upgrade)            shift; upgrade_package "$@" ;;
    *)                  echo "Unrecognized command: '$1'" >&2
                        echo "See 'spackle --help' for usage" >&2
esac