Added multiple QoL improvements + fixed slow download speeds

PS3GUD.py

@@ -21,7 +21,7 @@
 import json
 import time
 import platform
-import ssl
+from urllib3.exceptions import InsecureRequestWarning
 
 #local files
 import utils
@@ -31,6 +31,8 @@
 import requests.adapters
 import keyring
 
+requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
+
 class PS3GUD():
     def __init__(self, window=None):
         if window != None:
@@ -44,10 +46,6 @@ def __init__(self, window=None):
         self.DlList = Queue(self)
         self.titleid = ""
         self.proxies = {}
-        #handle sonys weak cert for their https server
-        self.https_session = requests.Session()
-        self.https_session.mount('https://a0.ww.np.dl.playstation.net', SonySSLContextAdapter())
-        self.pemfile = "./sony.pem"
 
         self.useDefaultConfig = True
         self.configDefaults = {}
@@ -57,6 +55,8 @@ def __init__(self, window=None):
         self.configDefaults["storageThresholdNew"] = 5
         self.configDefaults["currentLoc"] = "en"
         self.configDefaults["checkForNewRelease"] = True
+        self.configDefaults["bypass_ssl"] = True
+        self.configDefaults["add_all_updates_to_queue_automatically"] = False
         self.configDefaults["use_proxy"] = False
         self.configDefaults["proxy_ip"] = ""
         self.configDefaults["proxy_port"] = ""
@@ -66,6 +66,11 @@ def __init__(self, window=None):
         self.configDefaults["rename_pkgs"] = True
         self.configDefaults["update_titledb"] = True
 
+    def setupRequests(self):
+        self.https_session = requests.Session()
+        self.https_session.mount('https://a0.ww.np.dl.playstation.net', SonySSLContextAdapter(bypassSSL=self.getConfig("bypass_ssl")))
+        self.pemfile = "./sony.pem"
+
     def setWindow(self, window):
         self.logger.window = window
 
@@ -104,6 +109,8 @@ def loadConfig(self):
             self.logger.log(self.loc.getKey("msg_noConfigFile"))
             self.config = self.configDefaults
 
+        self.setupRequests()
+
     def setConfig(self, config):
         self.config = config
         self.saveConfig()
@@ -113,6 +120,7 @@ def saveConfig(self):
             f.write(json.dumps(self.config, sort_keys=True, indent=4, ensure_ascii=False))
         self.logger.log(self.loc.getKey("msg_configFileSaved"))
         self.useDefaultConfig = False 
+        self.setupRequests()
 
     def setProxyCredentials(self, pwd, user):
         keyring.set_password("ps3gud", "proxy_pass", pwd)
@@ -192,7 +200,7 @@ def getUpdates(self):
     def checkForUpdates(self, titleid):    
         #check given id
         check = False
-        titleid = titleid.upper()
+        titleid = titleid.upper().replace("-", "")
         for item in self.titledb:
             if titleid == item["id"]:
                 check = True
@@ -208,9 +216,14 @@ def checkForUpdates(self, titleid):
         updates = []
         url = urllib.parse.urljoin(urllib.parse.urljoin("https://a0.ww.np.dl.playstation.net/tpl/np/", self.titleid+"/"), self.titleid+"-ver.xml")
         try:
-            resp = self.https_session.get(url, verify=self.pemfile, proxies=self.proxies)
+            if self.getConfig("bypass_ssl"):
+                verify_value = False
+            else:
+                verify_value = self.pemfile
+
+            resp = self.https_session.get(url, verify=verify_value, proxies=self.proxies)
         except requests.exceptions.ConnectionError:
-            self.logger.log(self.loc.getKey("msg_metaNotAvailable"), "e")
+            self.logger.log(f"{self.loc.getKey('msg_metaNotAvailable')} ({url})")
             if self.getConfig("use_proxy"):
                 self.logger.log(self.loc.getKey("msg_checkProxySettings"))
             self.titleid = ""
@@ -256,9 +269,9 @@ def downloadFiles(self, window):
             sha1 = dl["sha1"]
             size = dl["size"]
             id = dl["gameid"]
-            fdir = os.path.join(self.getConfig("dldir")+"/", utils.filterIllegalCharsFilename(self.getTitleNameFromId(id))+"["+id+"]/")
+            fdir = os.path.join(self.getConfig("dldir")+"/", utils.filterIllegalCharsFilename(self.getTitleNameFromId(id))+" ["+id+"]/")
             if self.getConfig("rename_pkgs") == True:
-                fname = os.path.join(fdir, utils.filterIllegalCharsFilename(self.getTitleNameFromId(id)+"_["+id+"]_"+dl["version"]+".pkg"))
+                fname = os.path.join(fdir, utils.filterIllegalCharsFilename(self.getTitleNameFromId(id)+" ["+id+"] "+dl["version"]+".pkg"))
             else:
                 fname = os.path.join(fdir, utils.filterIllegalCharsFilename(os.path.basename(url)))
 
@@ -317,7 +330,7 @@ def _download_file(self, url, local_filename, size, window, num):
         text = window["window_main_progress_label"]
         bar = window["window_main_progress_bar"]
         size = int(size)
-        chunk_size=8192
+        chunk_size=98304
         count = 0
         already_loaded = 0
         with requests.get(url, stream=True, proxies=self.proxies) as r:
@@ -461,14 +474,24 @@ def exportQueue(self, exportFile):
         for id, data in games.items():
             s += f"{self.ps3.getTitleNameFromId(id)} [{id}]:\n\n"
             for url, version in data:
-                s += f"\t{url+'#name='+urllib.parse.quote(utils.filterIllegalCharsFilename(self.ps3.getTitleNameFromId(id)+'_['+id+']_'+version+'.pkg'))}\n"
+                filename = urllib.parse.quote(utils.filterIllegalCharsFilename(self.ps3.getTitleNameFromId(id)+' ['+id+'] '+version+'.pkg'))
+                folder = urllib.parse.quote(utils.filterIllegalCharsFilename(self.ps3.getTitleNameFromId(id)+' ['+id+']'))
+                s += f"\t{url+'#folder='+folder+'#name='+filename}\n"
             s += "\n"
         with open(exportFile, "w", encoding="utf8") as f:
             f.write(s)
 
 class SonySSLContextAdapter(requests.adapters.HTTPAdapter):
+    def __init__(self, bypassSSL=False, **kwargs):
+        self.bypassSSL = bypassSSL
+        super(SonySSLContextAdapter, self).__init__(**kwargs)
+
     def init_poolmanager(self, *args, **kwargs):
         context = ssl.create_default_context()
-        context.set_ciphers('DEFAULT:@SECLEVEL=1')
+        if self.bypassSSL:
+            context.check_hostname = False
+            context.verify_mode = ssl.CERT_NONE
+        else:
+            context.set_ciphers('DEFAULT:@SECLEVEL=1')
         kwargs['ssl_context'] = context
-        return super(SonySSLContextAdapter, self).init_poolmanager(*args, **kwargs)
+        return super(SonySSLContextAdapter, self).init_poolmanager(*args, **kwargs)

README.md

@@ -15,7 +15,9 @@ Grab the latest release [here](https://github.com/shinrax2/PS3GameUpdateDownload
 Sony has announced that they will be closing the PS3 store on 02.07.21 and later reversed that decision. It seems that some game updates got deleted in the time between the announcement and its reversal. I recommend you backup your downloaded game updates.
 
 # JDownloader PKG Renaming
-If you want to use the queue export feature and download the updates with JDownloader import `ps3gud_jdownloader_renamepkgs.packagizer` into your packagizer settings
+If you want to use the queue export feature and download the updates with [JDownloader](https://jdownloader.org/download/index) import `ps3gud_jdownloader_renamepkgs.packagizer` into your packagizer settings and make sure to set the following rule order:
+
+![Main window](https://github.com/shinrax2/PS3GameUpdateDownloader/raw/master/screenshots/jdownloader_order.png)
 
 # Contributions
 Contributions in form of translations and bug fixes are welcome.

gui.py

@@ -199,6 +199,8 @@ def configWin(self, nocancel=False):
             [sg.Text(self.loc.getKey("window_config_storageThreshold_label"), size=(40,1)), sg.In(self.ps3.getConfig("storageThresholdNew"), key="storageThreshold")],
             [sg.Text(self.loc.getKey("window_config_currentLoc_label"), size=(40,1)), sg.OptionMenu(locChoices, size=(8, 15), key="currentLoc", default_value=self.loc.getKey("language_name"))],
             [sg.Text(self.loc.getKey("window_config_renamepkgs_label"), size=(40,1), tooltip=self.loc.getKey("window_config_renamepkgs_tooltip")), sg.Checkbox("", default=self.ps3.getConfig("rename_pkgs"), key="rename_pkgs", tooltip=self.loc.getKey("window_config_renamepkgs_tooltip"))],
+            [sg.Text(self.loc.getKey("window_config_add_all_updates_to_queue_automatically_label"), size=(40,1), tooltip=self.loc.getKey("window_config_add_all_updates_to_queue_automatically_tooltip")), sg.Checkbox("", default=self.ps3.getConfig("add_all_updates_to_queue_automatically"), key="add_all_updates_to_queue_automatically", tooltip=self.loc.getKey("window_config_add_all_updates_to_queue_automatically_tooltip"))],
+            [sg.Text(self.loc.getKey("window_config_bypass_ssl_label"), size=(40,1), tooltip=self.loc.getKey("window_config_bypass_ssl_tooltip")), sg.Checkbox("", default=self.ps3.getConfig("bypass_ssl"), key="bypass_ssl", tooltip=self.loc.getKey("window_config_bypass_ssl_tooltip"))],
             [sg.Text(self.loc.getKey("window_config_updatetitledb_label"), size=(40,1)), sg.Checkbox("", default=self.ps3.getConfig("update_titledb"), key="update_titledb")],
             [sg.Text(self.loc.getKey("window_config_useproxy_label"), size=(40,1)), sg.Checkbox("", default=self.ps3.getConfig("use_proxy"), key="use_proxy", enable_events=True)],
             [sg.Text(self.loc.getKey("window_config_proxyip_label"), size=(40,1)), sg.In(self.ps3.getConfig("proxy_ip"), key="proxy_ip")],
@@ -243,7 +245,7 @@ def configWin(self, nocancel=False):
                 for l in ll:
                     if cL == l["language_name"]:
                         cL = l["language_short"]
-                config = { "dldir": valConfig["dldir"], "verify": valConfig["verify"], "checkIfAlreadyDownloaded": valConfig["checkIfAlreadyDownloaded"], "storageThresholdNew": valConfig["storageThreshold"], "currentLoc": cL , "proxy_ip": valConfig["proxy_ip"], "proxy_port": valConfig["proxy_port"], "use_proxy": valConfig["use_proxy"], "rename_pkgs": valConfig["rename_pkgs"], "update_titledb": valConfig["update_titledb"]}
+                config = { "dldir": valConfig["dldir"], "verify": valConfig["verify"], "checkIfAlreadyDownloaded": valConfig["checkIfAlreadyDownloaded"], "storageThresholdNew": valConfig["storageThreshold"], "currentLoc": cL , "proxy_ip": valConfig["proxy_ip"], "proxy_port": valConfig["proxy_port"], "use_proxy": valConfig["use_proxy"], "rename_pkgs": valConfig["rename_pkgs"], "update_titledb": valConfig["update_titledb"], "bypass_ssl": valConfig["bypass_ssl"], "add_all_updates_to_queue_automatically": valConfig["add_all_updates_to_queue_automatically"]}
                 self.ps3.setConfig(config)
                 if self.ps3.getConfig("use_proxy") == True:
                     self.ps3.setProxyCredentials(valConfig["proxy_pass"], valConfig["proxy_user"])
@@ -261,61 +263,69 @@ def selectWin(self):
             updlen = 0
             self.tryDl = False
         if updlen > 0:
-            self.mainWindow.hide()
-            data = self.updatePackToTable(self.ps3.getUpdates())
-            lay2 = [
-                [sg.Text(self.loc.getKey("window_select_text_label", [self.ps3.getTitleNameFromId(), self.ps3.titleid]))],
-                [sg.Table(values=data, headings=[self.loc.getKey("window_select_table_ver"), self.loc.getKey("window_select_table_size"), self.loc.getKey("window_select_table_sysver")], key="Table", enable_events=True)],
-                [sg.Text(self.loc.getKey("window_select_help_label"), size=(45,2))],
-                [sg.Button(self.loc.getKey("window_select_download_btn"), key="OK", disabled=True),sg.Button(self.loc.getKey("window_select_queue_btn"), key="Queue", disabled=True), sg.Button(self.loc.getKey("window_select_cancel_btn"), key="Cancel")]
-            ]
-            self.selectWindow = sg.Window(self.loc.getKey("window_select_title"), lay2, icon=self.iconpath)
-            while True:
-                ev2, val2 = self.selectWindow.Read()
-                if len(val2["Table"]) > 0:
-                    self.selectWindow["OK"].Update(disabled=False)
-                    self.selectWindow["Queue"].Update(disabled=False)
-                if len(val2["Table"]) == 0:
-                    self.selectWindow["OK"].Update(disabled=True)
-                    self.selectWindow["Queue"].Update(disabled=True)
-                if ev2 in (None, "Exit"):
-                    self.selectWindow.Close()
-                    self.mainWindow.UnHide()
-                    self.tryDl = False
-                    break
-                if ev2 == "Cancel":
-                    self.selectWindow.Close()
-                    self.mainWindow.UnHide()
-                    self.tryDl = False
-                    break
-                if ev2 == "OK" and len(val2["Table"]) > 0:
-                    if len(val2["Table"]) == 1:
-                        self.ps3.DlList.addEntry(self.ps3.getUpdates()[val2["Table"][0]])
-                        self.selectWindow.Close()
-                        self.mainWindow.UnHide()
-                        self.tryDl = True
-                        break
-                    if len(val2["Table"]) > 1:
-                        for row in val2["Table"]:
-                            self.ps3.DlList.addEntry(self.ps3.getUpdates()[row])
-                        self.selectWindow.Close()
-                        self.mainWindow.UnHide()
-                        self.tryDl = True
-                        break
-                if ev2 == "Queue" and len(val2["Table"]) > 0:
-                    if len(val2["Table"]) == 1:    
-                        self.ps3.DlList.addEntry(self.ps3.getUpdates()[val2["Table"][0]])
+            updates = self.ps3.getUpdates()
+
+            if self.ps3.getConfig("add_all_updates_to_queue_automatically"):
+                for update in updates:
+                    self.ps3.DlList.addEntry(update)
+
+                self.ps3.logger.log(f"Added {updlen} updates to queue")
+            else:
+                self.mainWindow.hide()
+                data = self.updatePackToTable(updates)
+                lay2 = [
+                    [sg.Text(self.loc.getKey("window_select_text_label", [self.ps3.getTitleNameFromId(), self.ps3.titleid]))],
+                    [sg.Table(values=data, headings=[self.loc.getKey("window_select_table_ver"), self.loc.getKey("window_select_table_size"), self.loc.getKey("window_select_table_sysver")], key="Table", enable_events=True)],
+                    [sg.Text(self.loc.getKey("window_select_help_label"), size=(45,2))],
+                    [sg.Button(self.loc.getKey("window_select_download_btn"), key="OK", disabled=True),sg.Button(self.loc.getKey("window_select_queue_btn"), key="Queue", disabled=True), sg.Button(self.loc.getKey("window_select_cancel_btn"), key="Cancel")]
+                ]
+                self.selectWindow = sg.Window(self.loc.getKey("window_select_title"), lay2, icon=self.iconpath)
+                while True:
+                    ev2, val2 = self.selectWindow.Read()
+                    if len(val2["Table"]) > 0:
+                        self.selectWindow["OK"].Update(disabled=False)
+                        self.selectWindow["Queue"].Update(disabled=False)
+                    if len(val2["Table"]) == 0:
+                        self.selectWindow["OK"].Update(disabled=True)
+                        self.selectWindow["Queue"].Update(disabled=True)
+                    if ev2 in (None, "Exit"):
                         self.selectWindow.Close()
                         self.mainWindow.UnHide()
                         self.tryDl = False
                         break
-                    if len(val2["Table"]) > 1:
-                        for row in val2["Table"]:
-                            self.ps3.DlList.addEntry(self.ps3.getUpdates()[row])
+                    if ev2 == "Cancel":
                         self.selectWindow.Close()
                         self.mainWindow.UnHide()
                         self.tryDl = False
                         break
+                    if ev2 == "OK" and len(val2["Table"]) > 0:
+                        if len(val2["Table"]) == 1:
+                            self.ps3.DlList.addEntry(self.ps3.getUpdates()[val2["Table"][0]])
+                            self.selectWindow.Close()
+                            self.mainWindow.UnHide()
+                            self.tryDl = True
+                            break
+                        if len(val2["Table"]) > 1:
+                            for row in val2["Table"]:
+                                self.ps3.DlList.addEntry(self.ps3.getUpdates()[row])
+                            self.selectWindow.Close()
+                            self.mainWindow.UnHide()
+                            self.tryDl = True
+                            break
+                    if ev2 == "Queue" and len(val2["Table"]) > 0:
+                        if len(val2["Table"]) == 1:    
+                            self.ps3.DlList.addEntry(self.ps3.getUpdates()[val2["Table"][0]])
+                            self.selectWindow.Close()
+                            self.mainWindow.UnHide()
+                            self.tryDl = False
+                            break
+                        if len(val2["Table"]) > 1:
+                            for row in val2["Table"]:
+                                self.ps3.DlList.addEntry(self.ps3.getUpdates()[row])
+                            self.selectWindow.Close()
+                            self.mainWindow.UnHide()
+                            self.tryDl = False
+                            break
 
 
     def queueWin(self):

loc/en.json

@@ -70,6 +70,22 @@
     "window_config_renamepkgs_tooltip": {
         "string": "Renames PKGs to $GAMENAME_[$GAMEID]_$GAMEVERSION",
         "comment": ""
+    },
+	"window_config_add_all_updates_to_queue_automatically_label": {
+        "string": "Add all updates to queue automatically:",
+        "comment": ""
+    },
+    "window_config_add_all_updates_to_queue_automatically_tooltip": {
+        "string": "Automatically add all game updates to queue without displaying the confirmation and selection window",
+        "comment": ""
+    },
+	"window_config_bypass_ssl_label": {
+        "string": "Bypass SSL:",
+        "comment": ""
+    },
+    "window_config_bypass_ssl_tooltip": {
+        "string": "Do not attempt to resolve and validate SSL when downloading",
+        "comment": ""
     },
     "window_config_updatetitledb_label": {
         "string": "Update TitleDB on startup:",