Fix #167 - PAC encoding support, fix #171

.gitignore

@@ -58,4 +58,5 @@ test-*/
 *.egg-info
 _*.stats
 *.dll
-*.log
+*.log
+*.crt

HISTORY.txt

@@ -1,6 +1,8 @@
-v0.8.4 - TBD
+v0.8.4 - 2023-02-06
+- Support for specifying PAC file encoding - #167
 - Fixed #164 - PAC function myIpAddress() was broken
 - Fixed #161 - PAC regex search was failing
+- Fixed #171 - Verbose output implies --foreground
 
 v0.8.3 - 2022-07-19
 - Fixed #157 - libcurl wrapper was missing socket definitions for OSX

README.md

@@ -220,6 +220,10 @@ Configuration:
     Use in place of --server if PAC file should be loaded from a URL or local
     file. Relative paths will be relative to the Px script or binary
 
+  --pac_encoding= proxy:pac_encoding=
+  PAC file encoding
+    Specify in case default 'utf-8' encoding does not work
+
   --listen=  proxy:listen=
   IP interface to listen on. Valid IP address, default: 127.0.0.1
 
@@ -301,8 +305,8 @@ Configuration:
     Px will attach to the console and write to it even though the prompt is
     available for further commands. CTRL-C in the console will exit Px
 
-  --verbose  settings:verbose=
-  Enable debug output. default: 0
+  --verbose
+  Enable debug output. default: 0. Implies --foreground
 
   --debug  settings:log=
   Enable debug logging. default: 0

build.ps1

@@ -34,6 +34,9 @@ Invoke-Expression "$PY -m pip install --upgrade nuitka twine"
 # Install wheel dependencies
 Invoke-Expression "$PY -m pip install --upgrade px-proxy --no-index -f px.dist-wheels-windows-amd64\px.dist-wheels"
 
+# Download libcurl
+Invoke-Expression "$PY tools.py --libcurl"
+
 # Build wheels
 Invoke-Expression "$PY tools.py --wheel"
 

build.sh

@@ -179,7 +179,7 @@ elif [ "$OS" = "Darwin" ]; then
         # Install Python
         brew install python@$pyver
 
-        PY="/usr/local/opt/python@$pyver/bin/python3"
+        PY="/usr/local/opt/python@$pyver/bin/python$pyver"
 
         # Tools
         $PY -m pip install --upgrade pip setuptools build wheel
@@ -190,6 +190,7 @@ elif [ "$OS" = "Darwin" ]; then
 
     # Install build tools
     $PY -m pip install --upgrade nuitka twine
+    brew install upx
 
     # Install wheel dependencies
     $PY -m pip install --upgrade px-proxy --no-index -f px.dist-wheels-osx-x86_64/px.dist-wheels

px.ini

@@ -10,6 +10,10 @@ server =
 ;   file. Relative paths will be relative to the Px script or binary
 pac =
 
+; PAC file encoding
+;   Specify in case default 'utf-8' encoding does not work
+; pac_encoding =
+
 ; IP interface to listen on - default: 127.0.0.1
 listen = 127.0.0.1
 

px/main.py

@@ -124,6 +124,10 @@
     Use in place of --server if PAC file should be loaded from a URL or local
     file. Relative paths will be relative to the Px script or binary
 
+  --pac_encoding= proxy:pac_encoding=
+  PAC file encoding
+    Specify in case default 'utf-8' encoding does not work
+
   --listen=  proxy:listen=
   IP interface to listen on - default: 127.0.0.1
 
@@ -205,8 +209,8 @@
     Px will attach to the console and write to it even though the prompt is
     available for further commands. CTRL-C in the console will exit Px
 
-  --verbose  settings:verbose=
-  Enable debug output. default: 0
+  --verbose
+  Enable debug output. default: 0. Implies --foreground
 
   --debug  settings:log=
   Enable debug logging. default: 0
@@ -702,6 +706,9 @@ def parse_config():
     elif "--verbose" in sys.argv:
         State.debug = Debug()
         dprint = State.debug.get_print()
+        if "--foreground" not in sys.argv:
+            # --verbose implies --foreground
+            sys.argv.append("--foreground")
 
     if sys.platform == "win32":
         if is_compiled() or "pythonw.exe" in sys.executable:
@@ -731,6 +738,7 @@ def parse_config():
 
     cfg_str_init("proxy", "server", "")
     cfg_str_init("proxy", "pac", "", set_pac)
+    cfg_str_init("proxy", "pac_encoding", "utf-8")
     cfg_int_init("proxy", "port", "3128")
     cfg_str_init("proxy", "listen", "127.0.0.1")
     cfg_str_init("proxy", "allow", "*.*.*.*", parse_allow)
@@ -765,6 +773,8 @@ def parse_config():
                 cfg_str_init("proxy", "server", val, None, True)
             elif "--pac=" in sys.argv[i]:
                 cfg_str_init("proxy", "pac", val, set_pac, True)
+            elif "--pac_encoding=" in sys.argv[i]:
+                cfg_str_init("proxy", "pac_encoding", val, None, True)
             elif "--listen=" in sys.argv[i]:
                 cfg_str_init("proxy", "listen", val, None, True)
             elif "--port=" in sys.argv[i]:
@@ -838,7 +848,8 @@ def parse_config():
     if len(servers) != 0:
         State.wproxy = wproxy.Wproxy(wproxy.MODE_CONFIG, servers, State.noproxy, debug_print = dprint)
     elif len(State.pac) != 0:
-        State.wproxy = wproxy.Wproxy(wproxy.MODE_CONFIG_PAC, [State.pac], debug_print = dprint)
+        pac_encoding = State.config.get("proxy", "pac_encoding")
+        State.wproxy = wproxy.Wproxy(wproxy.MODE_CONFIG_PAC, [State.pac], pac_encoding = pac_encoding, debug_print = dprint)
     else:
         State.wproxy = wproxy.Wproxy(debug_print = dprint)
         State.proxy_last_reload = time.time()

px/mcurl.py

@@ -462,19 +462,31 @@ def buffer(self, data = None):
 
         self.bridge(rfile, wfile, hfile)
 
-    def get_data(self):
-        "Return data written by curl perform to buffer()"
-        if isinstance(self.client_wfile, io.BytesIO):
-            return self.client_wfile.getvalue().decode("utf-8")
-        else:
-            return ""
+    def get_data(self, encoding = "utf-8"):
+        """
+        Return data written by curl perform to buffer()
 
-    def get_headers(self):
-        "Return headers written by curl to buffer()"
+        encoding = "utf-8" by default, change or set to None if bytes preferred
+        """
+        val = b""
         if isinstance(self.client_wfile, io.BytesIO):
-            return self.client_wfile.getvalue().decode("utf-8")
-        else:
-            return ""
+            val = self.client_wfile.getvalue()
+        if encoding is not None:
+            val = val.decode(encoding)
+        return val
+
+    def get_headers(self, encoding = "utf-8"):
+        """
+        Return headers written by curl perform to buffer()
+
+        encoding = "utf-8" by default, change or set to None if bytes preferred
+        """
+        val = b""
+        if isinstance(self.client_hfile, io.BytesIO):
+            val = self.client_hfile.getvalue()
+        if encoding is not None:
+            val = val.decode(encoding)
+        return val
 
     def set_transfer_decoding(self, enable = False):
         "Set curl to turn off transfer decoding - let client do it"

px/pac.py

@@ -37,20 +37,31 @@ def __init__(self, debug_print = None):
         # Load PAC js utils
         self.ctxt.eval(PACUTILS)
 
-    def load_jsfile(self, jsfile):
+    def load(self, pac_data, pac_encoding):
+        "Load PAC data in specified encoding into this context"
+        pac_encoding = pac_encoding or "utf-8"
+        text = pac_data.decode(pac_encoding)
+        try:
+            self.ctxt.eval(text)
+        except quickjs.JSException as exc:
+            dprint("PAC file parsing failed - syntax error or file not encoded in " + pac_encoding)
+            dprint("Use --pac_encoding or proxy:pac_encoding in px.ini to change")
+            raise exc
+
+    def load_jsfile(self, jsfile, pac_encoding):
         "Load specified JS file into this context"
         dprint("Loading PAC file: " + jsfile)
-        with open(jsfile) as js:
-            self.ctxt.eval(js.read())
+        with open(jsfile, "rb") as js:
+            self.load(js.read(), pac_encoding)
 
-    def load_url(self, jsurl):
+    def load_url(self, jsurl, pac_encoding):
         dprint("Loading PAC url: " + jsurl)
         c = Curl(jsurl)
         c.set_debug()
         c.buffer()
         c.set_follow()
         if c.perform():
-            self.ctxt.eval(c.get_data())
+            self.load(c.get_data(None), pac_encoding)
 
     def find_proxy_for_url(self, url, host):
         """

px/wproxy.py

@@ -131,15 +131,17 @@ class _WproxyBase:
     noproxy = None
     noproxy_hosts = None
     pac = None
+    pac_encoding = None
 
-    def __init__(self, mode = MODE_NONE, servers = None, noproxy = None, debug_print = None):
+    def __init__(self, mode = MODE_NONE, servers = None, noproxy = None, pac_encoding = None, debug_print = None):
         global dprint
         if debug_print is not None:
             dprint = debug_print
 
         self.servers = []
         self.noproxy = netaddr.IPSet([])
         self.noproxy_hosts = []
+        self.pac_encoding = pac_encoding
 
         if mode != MODE_NONE:
             # MODE_CONFIG or MODE_CONFIG_PAC
@@ -248,9 +250,9 @@ def find_proxy_for_url(self, url):
                 # Load PAC file
                 self.pac = Pac(debug_print = dprint)
                 if self.servers[0].startswith("http"):
-                    self.pac.load_url(self.servers[0])
+                    self.pac.load_url(self.servers[0], self.pac_encoding)
                 else:
-                    self.pac.load_jsfile(self.servers[0])
+                    self.pac.load_jsfile(self.servers[0], self.pac_encoding)
 
             if self.pac is not None:
                 return parse_proxy(self.pac.find_proxy_for_url(url, netloc[0])), netloc, path
@@ -323,7 +325,7 @@ class WINHTTP_PROXY_INFO(ctypes.Structure):
     class Wproxy(_WproxyBase):
         "Load proxy information from Windows Internet Options"
 
-        def __init__(self, mode = MODE_NONE, servers = None, noproxy = None, debug_print = None):
+        def __init__(self, mode = MODE_NONE, servers = None, noproxy = None, pac_encoding = None, debug_print = None):
             """
             Load proxy information from Windows Internet Options
               Returns MODE_NONE, MODE_ENV, MODE_AUTO, MODE_PAC, MODE_MANUAL
@@ -346,7 +348,7 @@ def __init__(self, mode = MODE_NONE, servers = None, noproxy = None, debug_print
 
             if mode != MODE_NONE:
                 # Check MODE_CONFIG and MODE_CONFIG_PAC cases
-                super().__init__(mode, servers, noproxy, debug_print)
+                super().__init__(mode, servers, noproxy, pac_encoding, debug_print)
 
             if self.mode == MODE_NONE:
                 # Get proxy info from Internet Options
@@ -392,7 +394,7 @@ def __init__(self, mode = MODE_NONE, servers = None, noproxy = None, debug_print
             dprint("Proxy mode = " + MODES[self.mode])
 
         # Find proxy for specified URL using WinHttp API
-        #   Used internally for MODE_AUTO, MODE_PAC and MODE_CONFIG_PAC
+        #   Used internally for MODE_AUTO and MODE_PAC
         def winhttp_find_proxy_for_url(self, url, autologon=True):
             ACCESS_TYPE = WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY
             if WIN_VERSION < 6.3:
@@ -408,7 +410,7 @@ def winhttp_find_proxy_for_url(self, url, autologon=True):
                 return ""
 
             autoproxy_options = WINHTTP_AUTOPROXY_OPTIONS()
-            if self.mode in [MODE_PAC, MODE_CONFIG_PAC]:
+            if self.mode == MODE_PAC:
                 autoproxy_options.dwFlags = WINHTTP_AUTOPROXY_CONFIG_URL
                 autoproxy_options.dwAutoDetectFlags = 0
                 autoproxy_options.lpszAutoConfigUrl = self.servers[0]
@@ -418,7 +420,7 @@ def winhttp_find_proxy_for_url(self, url, autologon=True):
                     WINHTTP_AUTO_DETECT_TYPE_DHCP | WINHTTP_AUTO_DETECT_TYPE_DNS_A)
                 autoproxy_options.lpszAutoConfigUrl = 0
             else:
-                dprint("winhttp_find_proxy_for_url only applicable for MODE_AUTO, MODE_PAC and MODE_CONFIG_PAC")
+                dprint("winhttp_find_proxy_for_url only applicable for MODE_AUTO and MODE_PAC")
                 return ""
             autoproxy_options.fAutoLogonIfChallenged = autologon
 

test.py

@@ -231,6 +231,9 @@ def getips():
 def checkPxStart(ip, port):
     # Make sure Px starts
     retry = 20
+    if sys.platform == "darwin":
+        # Nuitka builds take longer to start on OSX
+        retry = 40
     while True:
         try:
             socket.create_connection((ip, port), 2)
@@ -547,6 +550,7 @@ def auto():
         exec(cmd)
 
     if "--norun" not in sys.argv:
+        os.system("grep didn -i *.log")
         os.system("grep failed -i *.log")
         os.system("grep traceback -i *.log")
         os.system("grep error -i *.log")

tools.py

@@ -8,7 +8,11 @@
 import time
 import zipfile
 
-from px import mcurl
+if "--libcurl" not in sys.argv:
+    from px import mcurl
+else:
+    import urllib.request
+
 from px.version import __version__
 
 REPO = "genotrance/px"
@@ -146,8 +150,7 @@ def get_curl():
             lcurlzip = "curl%s.zip" % bit
             if not os.path.exists(lcurl):
                 if not os.path.exists(lcurlzip):
-                    with open(lcurlzip, "wb") as lcz:
-                        ret, _ = curl("https://curl.se/windows/curl-win%s-latest.zip" % bit, wfile = lcz)
+                    urllib.request.urlretrieve("https://curl.se/windows/latest.cgi?p=win%s-mingw.zip" % bit, lcurlzip)
                 extract(lcurlzip, lcurl)
 
                 if not os.path.exists("curl-ca-bundle.crt"):
@@ -173,8 +176,6 @@ def get_curl():
 def wheel():
     rmtree("build wheel")
 
-    get_curl()
-
     for args in ["--universal", "-p win32", "-p win-amd64"]:
         while True:
             rmtree("build")
@@ -228,6 +229,7 @@ def nuitka():
         if sys.platform == "win32":
             os.system("upx --best px.exe python3*.dll libcrypto*.dll")
         else:
+            os.rename("px.bin", "px")
             os.system("upx --best px")
 
     # Create archive