From fcb7b6f56af2106ebdfd3df0f856c074d70beaba Mon Sep 17 00:00:00 2001 From: Ganesh Viswanathan Date: Tue, 28 Jun 2022 15:20:37 -0500 Subject: [PATCH] Fix #154, linux deps, offline install --- HISTORY.txt | 8 +++ README.md | 40 ++++++++++--- build.ps1 | 44 +++++++++++++++ build.sh | 143 +++++++++++++++++++++++++++++++++++++--------- px/mcurl.py | 71 +++++++++++++---------- px/version.py | 2 +- setup.py | 5 +- test.py | 29 ++++++++-- tools.py | 153 ++++++++++++++++++++++++++++++++++++++++---------- 9 files changed, 393 insertions(+), 102 deletions(-) create mode 100644 build.ps1 diff --git a/HISTORY.txt b/HISTORY.txt index 1e168ed..d2b64e8 100644 --- a/HISTORY.txt +++ b/HISTORY.txt @@ -1,3 +1,11 @@ +v0.8.1 - 2022-06-27 +- Fixed #154 - improved SSL connection handling with libcurl by querying active + socket to get the sock_fd in select() instead of relying on sockopt_callback() + which does not get called when connections get reused +- Fixed keyring dependencies on Linux +- Added infrastructure to generate and post binary wheels for Px and all its + dependencies for offline installation + v0.8.0 - 2022-06-18 - Added PAC file support for Linux - Local PAC files on Windows are now processed using QuickJS instead of WinHttp diff --git a/README.md b/README.md index b6acda1..f1cef65 100644 --- a/README.md +++ b/README.md @@ -17,20 +17,36 @@ supported by [libcurl](https://curl.se/libcurl/c/CURLOPT_HTTPAUTH.html). ## Installation -If Python is already available, Px can be easily installed using the Python -package manager `pip`. This will download and install Px as a Python module -along with all dependencies: +The whole point of Px is to help tools get through a typical corporate proxy. +This means using a package manager to install Px might not always be feasible +which is why Px offers two binary options: +- If Python is already available, Px and all its dependencies can be easily +installed by downloading the `wheels` package for the target OS from the +[releases](https://github.com/genotrance/px/releases) page. After extraction, +Px and all dependencies can be installed with `pip`: + + python -m pip install px-proxy --no-index -f /path/to/wheels + +- If Python is not available, get the latest compiled binary from the +[releases](https://github.com/genotrance/px/releases) page instead. These +binaries are compiled with [Nuitka](https://nuitka.net) and contain everything +needed to run standalone. + +If direct internet access is available along with Python, Px can be easily +installed using the Python package manager `pip`. This will download and install +Px as a Python module along with all dependencies: python -m pip install px-proxy +On Windows, `scoop` can also be used to install Px: + + scoop install px + Once installed, Px can be run as follows: - Running `px` directly - In the background: `pythonw -m px` - In the foreground in a console window: `python -m px` -If Python is not available, get the latest binary from the [releases](https://github.com/genotrance/px/releases) -page. These binaries are compiled with [Nuitka](https://nuitka.net) and contain -everything needed to run standalone. Px requires [libcurl](https://curl.se/libcurl/) and the Windows builds ship with a copy. On Linux, it is required to install libcurl using the package manager: @@ -60,8 +76,12 @@ dependencies : python -m pip install . -Note that libcurl will need to be installed on Linux, as described earlier, -using the package manager. For Windows, [download](https://curl.se/windows/) and +NOOTE: Source install methods will require internet access since Python will try +to install Px dependencies from the internet. The binaries mentioned in the +previous section could be used to bootstrap a source install. + +NOTE: libcurl will need to be installed on Linux, as described earlier, using +the package manager. For Windows, [download](https://curl.se/windows/) and extract `libcurl.dll` and `libcurl-x64.dll` to `$PATH`. ### Without installation @@ -349,6 +369,10 @@ the following Python packages: - [netaddr](https://pypi.org/project/netaddr/) - [psutil](https://pypi.org/project/psutil/) - [quickjs](https://pypi.org/project/quickjs/) +- Linux + - [jeepney](https://pypi.org/project/jeepney/) + - [keyring_jeepney](https://pypi.org/project/keyring_jeepney/) + - [keyrings.alt](https://pypi.org/project/keyrings.alt/) - [futures](https://pypi.org/project/futures/) on Python 2.x Px also depends on [libcurl](https://curl.se/libcurl) for all outbound HTTP diff --git a/build.ps1 b/build.ps1 new file mode 100644 index 0000000..6ae8b1a --- /dev/null +++ b/build.ps1 @@ -0,0 +1,44 @@ +# Install scoop +if ((Get-Command "scoop" -ErrorAction SilentlyContinue) -eq $null) { + Set-ExecutionPolicy RemoteSigned -Scope CurrentUser + irm get.scoop.sh | iex +} + +# Install versions bucket +scoop bucket add versions -ErrorAction SilentlyContinue + +# Python versions +$PYVERSIONS = @("37", "38", "39", "310") +$PY = "python310" + +# Delete depspkg directory +Remove-Item -Recurse -Force px.dist-wheels-windows-amd64 + +# Setup Python and dependencies, build wheels for Px +foreach ($pyver in $PYVERSIONS) { + if ((Get-Command "python$pyver" -ErrorAction SilentlyContinue) -eq $null) { + # Install Python + scoop install python$pyver + } + + # Tools + Invoke-Expression "python$pyver -m pip install --upgrade pip setuptools build wheel" + + # Create wheel dependencies for this Python version + Invoke-Expression "python$pyver tools.py --deps" +} + +# Install build tools +Invoke-Expression "$PY -m pip install --upgrade nuitka twine" + +# Install wheel dependencies +Invoke-Expression "$PY -m pip install --upgrade px-proxy --no-index -f px.dist-wheels-windows-amd64\px.dist-wheels" + +# Build wheels +Invoke-Expression "$PY tools.py --wheel" + +# Create package of all dependencies +Invoke-Expression "$PY tools.py --depspkg" + +# Build Nuitka +Invoke-Expression "$PY tools.py --nuitka" \ No newline at end of file diff --git a/build.sh b/build.sh index fbbeffc..2a5a897 100644 --- a/build.sh +++ b/build.sh @@ -1,5 +1,23 @@ #! /bin/sh +# Usage +# +# Build wheels of all dependencies across glibc and musl +# ./build.sh deps +# +# Build nuitka binaries for glibc and musl +# ./build.sh nuitka +# +# Run test across all distros +# ./build.sh test +# +# Run command on specific container +# ./build.sh IMAGE command subcommand +# +# Commands +# build - sub-commands: deps nuitka +# test + if [ -f "/.dockerenv" ]; then # Running inside container DISTRO=`cat /etc/os-release | grep ^ID | head -n 1 | cut -d"=" -f2 | sed 's/"//g'` @@ -8,52 +26,68 @@ if [ -f "/.dockerenv" ]; then export PROXY="$1" export PAC="$2" export USERNAME="$3" + export AUTH="" - # build or test - COMMAND="test" + # build or test - else $SHELL + COMMAND="" if [ ! -z "$4" ]; then COMMAND="$4" fi + # build sub-commands: nuitka or deps + SUBCOMMAND="" + if [ ! -z "$5" ]; then + SUBCOMMAND="$5" + fi + if [ "$DISTRO" = "alpine" ]; then apk update && apk upgrade - apk add curl dbus gcc gnome-keyring linux-headers musl-dev psmisc python3 python3-dev + apk add curl psmisc python3 python3 -m ensurepip if [ "$COMMAND" = "build" ]; then - apk add ccache libffi-dev patchelf py3-dbus upx + apk add ccache gcc musl-dev patchelf python3-dev upx + elif [ "$COMMAND" = "test" ]; then + apk add dbus gnome-keyring fi SHELL="sh" elif [ "$DISTRO" = "centos" ]; then + # Avoid random mirror + cd /etc/yum.repos.d + for file in `ls`; do sed -i~ 's/^mirrorlist/#mirrorlist/' $file; done + for file in `ls`; do sed -i~~ 's/^#baseurl/baseurl/' $file; done + cd + yum update -y - yum install -y gnome-keyring psmisc + yum install -y psmisc python3 + python3 -m ensurepip if [ "$COMMAND" = "build" ]; then - yum install -y ccache dbus-devel libffi-devel patchelf python36-cryptography upx + yum install -y ccache libffi-devel patchelf python3-devel upx + elif [ "$COMMAND" = "test" ]; then + yum install -y gnome-keyring fi elif [ "$DISTRO" = "ubuntu" ] || [ "$DISTRO" = "debian" ] || [ "$DISTRO" = "linuxmint" ]; then apt update -y && apt upgrade -y - apt install -y curl dbus gnome-keyring psmisc python3 python3-dev python3-pip - if [ "$COMMAND" = "build" ]; then - apt install -y ccache patchelf python3-dbus python3-secretstorage upx zlib1g zlib1g-dev - fi + apt install -y curl dbus gnome-keyring psmisc python3 python3-pip + + export AUTH="--auth=NONEGOTIATE" elif [ "$DISTRO" = "opensuse-tumbleweed" ] || [ "$DISTRO" = "opensuse-leap" ]; then zypper -n update zypper -n install curl dbus-1 gnome-keyring psmisc python3 python3-pip - if [ "$DISTRO" = "opensuse-leap" ]; then - zypper -n install gcc - fi + + export AUTH="--auth=NONEGOTIATE" elif [ "$DISTRO" = "void" ]; then xbps-install -Suy xbps - xbps-install -Sy curl dbus gcc gnome-keyring psmisc python3 python3-devel + xbps-install -Sy curl dbus gnome-keyring psmisc python3 python3 -m ensurepip SHELL="sh" @@ -66,27 +100,59 @@ if [ -f "/.dockerenv" ]; then MUSL=`ldd /bin/ls | grep musl` if [ -z "$MUSL" ]; then - PXBIN="/px/px.dist-linux-glibc-x86_64/px.dist/px" + ABI="glibc" else - PXBIN="/px/px.dist-linux-musl-x86_64/px.dist/px" + ABI="musl" fi + export PXBIN="/px/px.dist-linux-$ABI-x86_64/px.dist/px" + export WHEELS="/px/px.dist-wheels-linux-$ABI-x86_64/px.dist-wheels" - dbus-run-session -- $SHELL -c 'echo "abc" | gnome-keyring-daemon --unlock' + if [ "$COMMAND" = "test" ]; then + dbus-run-session -- $SHELL -c 'echo "abc" | gnome-keyring-daemon --unlock' + fi cd /px if [ "$COMMAND" = "build" ]; then - python3 -m pip install --upgrade pip setuptools jeepney==0.7.1 - - python3 tools.py --setup - - python3 px.py --username=$USERNAME --password + if [ "$SUBCOMMAND" = "nuitka" ]; then + # Install tools + python3 -m pip install --upgrade pip setuptools build wheel + python3 -m pip install --upgrade nuitka + + # Install wheel dependencies + # nuitka depends on deps - run deps first + python3 -m pip install px-proxy --no-index -f $WHEELS + + # Build Nuitka binary + python3 tools.py --nuitka + elif [ "$SUBCOMMAND" = "deps" ]; then + rm -rf $WHEELS + + # Run for all Python versions + for pyver in `ls /opt/python/cp* -d` + do + # Install tools + $pyver/bin/python3 -m pip install --upgrade pip setuptools build wheel + + # Build dependency wheels + $pyver/bin/python3 tools.py --deps + done + + # Package all wheels + /opt/python/cp310-cp310/bin/python3 tools.py --depspkg + else + $SHELL + fi else - python3 -m pip install --upgrade pip setuptools netifaces psutil + python3 -m pip install --upgrade pip + python3 -m pip install px-proxy --no-index -f $WHEELS - $PXBIN --username=$USERNAME --password + if [ "$COMMAND" = "test" ]; then + python3 test.py --binary --pip --proxy=$PROXY --pac=$PAC --username=$USERNAME $AUTH + else + $SHELL + fi fi - $SHELL else # Start container if [ -z "$PROXY" ]; then @@ -104,5 +170,30 @@ else exit fi - docker run -it --rm --network host --privileged -v `pwd`:/px $1 /px/build.sh "$PROXY" "$PAC" "$USERNAME" $2 + # Python wheel containers for musl and glibc + MUSL="quay.io/pypa/musllinux_1_1_x86_64" + GLIBC="quay.io/pypa/manylinux2014_x86_64" + + DOCKERCMD="docker run -it --rm --network host --privileged -v `pwd`:/px -v /root/.local/share:/root/.local/share" + if [ "$1" = "deps" ] || [ "$1" = "nuitka" ]; then + for image in $MUSL $GLIBC + do + $DOCKERCMD $image /px/build.sh "$PROXY" "$PAC" "$USERNAME" build $1 + done + elif [ "$1" = "test" ]; then + for image in alpine alpine:3.11 voidlinux/voidlinux-musl ubuntu ubuntu:bionic debian debian:oldstable linuxmintd/mint20.3-amd64 opensuse/tumbleweed opensuse/leap:15.1 + do + $DOCKERCMD $image /px/build.sh "$PROXY" "$PAC" "$USERNAME" test + done + else + if [ "$1" = "musl" ]; then + IMAGE="$MUSL" + elif [ "$1" = "glibc" ]; then + IMAGE="$GLIBC" + else + IMAGE="$1" + fi + + $DOCKERCMD $IMAGE /px/build.sh "$PROXY" "$PAC" "$USERNAME" $2 $3 + fi fi \ No newline at end of file diff --git a/px/mcurl.py b/px/mcurl.py index cb99af3..ca28a81 100644 --- a/px/mcurl.py +++ b/px/mcurl.py @@ -324,6 +324,15 @@ def get_response(self): ret = libcurl.easy_getinfo(self.easy, libcurl.CURLINFO_RESPONSE_CODE, ctypes.byref(codep)) return ret, codep.value + def get_activesocket(self): + "Return active socket for this easy instance" + if sys.platform == "win32": + sock_fd = ctypes.c_uint() + else: + sock_fd = ctypes.c_int() + ret = libcurl.easy_getinfo(self.easy, libcurl.CURLINFO_ACTIVESOCKET, ctypes.byref(sock_fd)) + return ret, sock_fd.value + def set_proxy(self, proxy, port = 0, noproxy = None): """ Set proxy options - returns False if this proxy server has auth failures @@ -518,16 +527,14 @@ def _timer_callback(multi, timeout_ms, userp): @libcurl.sockopt_callback def _sockopt_callback(clientp, sock_fd, purpose): - # Associate new socket with last handle added - #dprint("sock_fd = %d" % sock_fd) - del clientp, purpose - if MCURL.last is not None: - MCURL.last.sock_fd = sock_fd - MCURL.last = None + # Associate new socket with easy handle + del purpose + curl = MCURL.handles[libcurl.from_oid(clientp)] + curl.sock_fd = sock_fd return libcurl.CURLE_OK -def curl_version(): +def print_curl_version(): "Display curl version information" dprint(libcurl.version().decode("utf-8")) vinfo = libcurl.version_info(libcurl.CURLVERSION_NOW).contents @@ -541,6 +548,9 @@ def curl_version(): dprint("%s: %s" % (feature, avail)) dprint("Host: " + vinfo.host.decode("utf-8")) +def curl_version(): + return libcurl.version_info(libcurl.CURLVERSION_NOW).contents.version_num + class MCurl: "Helper class to manage a curl multi instance" @@ -550,7 +560,6 @@ class MCurl: handles = None proxytype = None failed = None # Proxy servers with auth failures - last = None timer = None rlist = None wlist = None @@ -565,7 +574,7 @@ def __init__(self, debug_print = None): global MCURL MCURL = self - curl_version() + print_curl_version() self._multi = libcurl.multi_init() # Set a callback for registering or unregistering socket events. @@ -626,31 +635,22 @@ def _add_handle(self, curl: Curl): dprint(curl.easyhash + ": Add handle") if curl.easyhash not in self.handles: self.handles[curl.easyhash] = curl - if curl.is_connect(): + if curl.is_connect() and curl_version() < 0x072D00: # Need to know socket assigned for CONNECT since used later in select() - self.last = curl + # CURLINFO_ACTIVESOCKET not available on libcurl < v7.45 so need this + # hack for older versions libcurl.easy_setopt(curl.easy, libcurl.CURLOPT_SOCKOPTFUNCTION, _sockopt_callback) + libcurl.easy_setopt(curl.easy, libcurl.CURLOPT_SOCKOPTDATA, id(curl.easyhash)) libcurl.multi_add_handle(self._multi, curl.easy) dprint(curl.easyhash + ": Added handle") else: dprint(curl.easyhash + ": Active handle") def add(self, curl: Curl): - """ - Add a Curl handle to perform - - Waits until multi instance is ready to add a handle - """ - while True: - with self._lock: - dprint(curl.easyhash + ": Handles = %d" % len(self.handles)) - if self.last is not None: - dprint(curl.easyhash + ": Multi not ready for new curl") - else: - self._add_handle(curl) - break - self.perform() - time.sleep(0.01) + "Add a Curl handle to perform" + with self._lock: + dprint(curl.easyhash + ": Handles = %d" % len(self.handles)) + self._add_handle(curl) # Removing from multi @@ -668,7 +668,7 @@ def _remove_handle(self, curl: Curl, errstr = ""): dprint(curl.easyhash + ": Remove handle: " + curl.errstr) if len(curl.errstr) == 0: libcurl.multi_remove_handle(self._multi, curl.easy) - curl.sock_fd = None + self.handles.pop(curl.easyhash) def remove(self, curl: Curl): @@ -745,9 +745,20 @@ def select(self, curl: Curl, client_sock, idle = 30): "Run select loop between client and curl" # TODO figure out if IPv6 or IPv4 if curl.sock_fd is None: - # Reusing a CONNECT!? - dprint(curl.easyhash + ": sock_fd is None") - return + if curl_version() < 0x072D00: + # Reusing an SSL connection but no way to get active socket since + # CURLINFO_ACTIVESOCKET was only added in libcurl v7.45 + dprint(curl.easyhash + ": unable to reuse SSL connection with libcurl < v7.45") + return + + # Need to get the active socket using getinfo() + dprint(curl.easyhash + ": Getting active socket") + ret, sock_fd = curl.get_activesocket() + if ret == libcurl.CURLE_OK: + curl.sock_fd = sock_fd + else: + dprint(curl.easyhash + ": Failed to get active socket: %d, %d" % (ret, sock_fd)) + return dprint(curl.easyhash + ": Starting select loop") curl_sock = socket.fromfd(curl.sock_fd, socket.AF_INET, socket.SOCK_STREAM) diff --git a/px/version.py b/px/version.py index 1953bb3..a634e05 100644 --- a/px/version.py +++ b/px/version.py @@ -1,3 +1,3 @@ "Px version" -__version__ = "0.8.0" +__version__ = "0.8.1" diff --git a/setup.py b/setup.py index eee2ff3..b9435a6 100644 --- a/setup.py +++ b/setup.py @@ -78,7 +78,10 @@ "keyring", "netaddr", "psutil", - "quickjs" + "quickjs", + 'jeepney==0.7.1;platform_system=="Linux"', + 'keyring_jeepney==0.2;platform_system=="Linux"', + 'keyrings.alt;platform_system=="Linux"' ], data_files = data_files, entry_points = { diff --git a/test.py b/test.py index 97eddae..2c15b07 100644 --- a/test.py +++ b/test.py @@ -13,7 +13,9 @@ import psutil if sys.platform == "linux": - import netifaces + import fcntl + import struct + import array from px.version import __version__ @@ -217,10 +219,27 @@ def runTest(test, cmd, offset, port): def getips(): ip_list = [] if sys.platform == "linux": - for interface in netifaces.interfaces(): - for link in netifaces.ifaddresses(interface)[socket.AF_INET]: - ip_list.append(link['addr']) - return ip_list + # https://gist.github.com/pklaus/289646 + max_possible = 128 # arbitrary. raise if needed. + obytes = max_possible * 32 + deb = b'\0' + + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + names = array.array('B', deb * obytes) + outbytes = struct.unpack('iL', fcntl.ioctl( + s.fileno(), + 0x8912, # SIOCGIFCONF + struct.pack('iL', obytes, names.buffer_info()[0]) + ))[0] + + namestr = names.tobytes() + + for i in range(0, outbytes, 40): + name = namestr[ i: i+16 ].split( deb, 1)[0] + name = name.decode() + #iface_name = namestr[ i : i+16 ].split( deb, 1 )[0] + ip = namestr[i+20:i+24] + ip_list.append(".".join([str(i) for i in ip])) elif sys.platform == "win32": ip_list = [ip[4][0] for ip in socket.getaddrinfo(socket.gethostname(), 80, socket.AF_INET)] ip_list.insert(0, "127.0.0.1") diff --git a/tools.py b/tools.py index 637f9ef..3b8f638 100644 --- a/tools.py +++ b/tools.py @@ -37,10 +37,9 @@ def get_auth(): def rmtree(dirs): for dir in dirs.split(" "): - shutil.rmtree(dir, True) - time.sleep(0.1) - if os.path.exists(dir): + while os.path.exists(dir): shutil.rmtree(dir, True) + time.sleep(0.2) def copy(files, dir): for file in files.split(" "): @@ -60,13 +59,13 @@ def remove(files): except: pass -def extract(zfile, cfile): +def extract(zfile, fileend): with zipfile.ZipFile(zfile) as czip: for file in czip.namelist(): - if file.endswith(cfile): + if file.endswith(fileend): member = czip.open(file) - with open(cfile, "wb") as dll: - shutil.copyfileobj(member, dll) + with open(os.path.basename(file), "wb") as base: + shutil.copyfileobj(member, base) # OS @@ -83,6 +82,13 @@ def get_os(): return "unsupported" +def get_dirs(prefix): + osname = get_os() + outdir = "%s-%s-%s" % (prefix, osname, platform.machine().lower()) + dist = os.path.join(outdir, prefix) + + return osname, outdir, dist + # URL def curl(url, method = "GET", proxy = None, headers = None, data = None, rfile = None, rfile_size = 0, wfile = None): @@ -169,11 +175,12 @@ def wheel(): get_curl() - os.system(sys.executable + " setup.py bdist_wheel --universal -d wheel -k") - rmtree("build") - os.system(sys.executable + " setup.py bdist_wheel -p win32 -d wheel -k") - rmtree("build") - os.system(sys.executable + " setup.py bdist_wheel -p win-amd64 -d wheel -k") + for args in ["--universal", "-p win32", "-p win-amd64"]: + while True: + rmtree("build") + if os.system(sys.executable + " setup.py bdist_wheel -d wheel -k %s" % args) == 0: + break + time.sleep(0.5) # Check wheels os.system(sys.executable + " -m twine check wheel/*") @@ -181,8 +188,7 @@ def wheel(): rmtree("build px_proxy.egg-info") def pyinstaller(): - osname = get_os() - dist = "pyinst-%s-%s" % (osname, platform.machine().lower()) + _, dist, _ = get_dirs("pyinst") rmtree("build dist " + dist) os.system("pyinstaller --clean --noupx -w px.py") @@ -194,10 +200,9 @@ def pyinstaller(): os.rename("dist", dist) def nuitka(): - osname = get_os() - outdir = "px.dist-%s-%s" % (osname, platform.machine().lower()) - dist = os.path.join(outdir, "px.dist") - shutil.rmtree(outdir, True) + prefix = "px.dist" + osname, outdir, dist = get_dirs(prefix) + rmtree(outdir) # Build flags = "" @@ -228,7 +233,96 @@ def nuitka(): arch = "gztar" if sys.platform == "win32": arch = "zip" - shutil.make_archive(archfile, arch, "px.dist") + shutil.make_archive(archfile, arch, prefix) + + # Create hashfile + if arch == "gztar": + arch = "tar.gz" + archfile += "." + arch + with open(archfile, "rb") as afile: + sha256sum = hashlib.sha256(afile.read()).hexdigest() + with open(archfile + ".sha256", "w") as shafile: + shafile.write(sha256sum) + + os.chdir("..") + +def deps(): + prefix = "px.dist-wheels" + _, outdir, dist = get_dirs(prefix) + if "--force" in sys.argv: + rmtree(outdir) + + try: + os.mkdir(outdir) + except: + pass + try: + os.mkdir(dist) + except: + pass + + # Build + os.system(sys.executable + " -m pip wheel . -w " + dist) + +def depspkg(): + prefix = "px.dist-wheels" + osname, outdir, dist = get_dirs(prefix) + + if sys.platform == "linux": + # Use strings on Linux to reduce wheel size + # Not effective on Windows + os.chdir(dist) + + for whl in glob.glob("*.whl"): + size = os.stat(whl).st_size + wdir = os.path.basename(whl[:-4]) + os.system(sys.executable + " -m zipfile -e " + whl + " " + wdir) + + os.chdir(wdir) + processed = False + for so in glob.glob("**/*.so", recursive = True): + processed = True + strip = os.system("strip -s " + so) + if strip != 0: + processed = False + break + + os.chdir("..") + + if processed: + os.system(sys.executable + " -m zipfile -c " + whl + ".new " + wdir + "/*") + new_size = os.stat(whl + ".new").st_size + if new_size < size: + print("%s: size changed from %d to %d" % (whl, size, new_size)) + os.remove(whl) + os.rename(whl + ".new", whl) + else: + os.remove(whl + ".new") + + rmtree(wdir) + + os.chdir("..") + else: + os.chdir(outdir) + + # Replace with official Px wheel + try: + os.remove(os.path.join(prefix, "px_proxy-%s-py3-none-any.whl" % __version__)) + except: + pass + whl = "px_proxy-" + __version__ + if sys.platform == "win32": + whl += "-py3-none-win_amd64.whl" + else: + whl += "-py2.py3-none-any.whl" + shutil.copy(os.path.join("..", "wheel", whl), prefix) + + # Compress all wheels + archfile = "px-v%s-%s-wheels" % (__version__, osname) + arch = "gztar" + if sys.platform == "win32": + arch = "zip" + shutil.make_archive(archfile, arch, prefix) # Create hashfile if arch == "gztar": @@ -410,17 +504,7 @@ def post(): # Main def main(): # Setup - if "--setup" in sys.argv: - os.system(sys.executable + " -m pip install --upgrade keyring netaddr psutil quickjs") - os.system(sys.executable + " -m pip install --upgrade build nuitka twine wheel") - if sys.platform == "linux": - os.system(sys.executable + " -m pip install --upgrade keyrings.alt keyring_jeepney netifaces") - - get_curl() - - sys.exit() - - elif "--libcurl" in sys.argv: + if "--libcurl" in sys.argv: get_curl() sys.exit() @@ -436,6 +520,12 @@ def main(): if "--nuitka" in sys.argv: nuitka() + if "--deps" in sys.argv: + deps() + + if "--depspkg" in sys.argv: + depspkg() + # Delete if "--delete" in sys.argv: tag = get_argval("tag") or "v" + __version__ @@ -461,13 +551,14 @@ def main(): if len(sys.argv) == 1: print(""" Setup: ---setup Install all px runtime and development dependencies --libcurl Download and extract libcurl binaries for Windows Build: --wheel Build wheels for pypi.org --pyinst Build px.exe using PyInstaller --nuitka Build px distribution using Nuitka +--deps Build all wheel dependencies for this Python version +--depspkg Build an archive of all dependencies Post: --twine Post wheels to pypi.org