Add an ini file to configure certain settings

ao3downloader/actions/ao3download.py

@@ -6,8 +6,8 @@
 
 
 def action():
-    with Repository() as repo:
-        fileops = FileOps()
+    fileops = FileOps()
+    with Repository(fileops) as repo:
 
         filetypes = shared.download_types(fileops)
         series = shared.series()

ao3downloader/actions/enterlinks.py

@@ -7,8 +7,8 @@
 from tqdm import tqdm
 
 def action():
-    with Repository() as repo:
-        fileops = FileOps()
+    fileops = FileOps()
+    with Repository(fileops) as repo:
 
         filetypes = shared.download_types(fileops)
         images = shared.images()

ao3downloader/actions/getlinks.py

@@ -10,8 +10,8 @@
 
 
 def action():
-    with Repository() as repo:
-        fileops = FileOps()
+    fileops = FileOps()
+    with Repository(fileops) as repo:
 
         link = shared.link(fileops)
         series = shared.series()

ao3downloader/actions/markedforlater.py

@@ -6,8 +6,8 @@
 
 
 def action():
-    with Repository() as repo:
-        fileops = FileOps()
+    fileops = FileOps()
+    with Repository(fileops) as repo:
 
         filetypes = shared.download_types(fileops)
         series = shared.series()

ao3downloader/actions/pinboarddownload.py

@@ -7,8 +7,8 @@
 
 
 def action():
-    with Repository() as repo:
-        fileops = FileOps()
+    fileops = FileOps()
+    with Repository(fileops) as repo:
 
         filetypes = shared.download_types(fileops)
         date = shared.pinboard_date()

ao3downloader/actions/redownload.py

@@ -9,8 +9,8 @@
 
 
 def action():
-    with Repository() as repo:
-        fileops = FileOps()
+    fileops = FileOps()
+    with Repository(fileops) as repo:
 
         folder = shared.redownload_folder()
         oldtypes = shared.redownload_oldtypes()

ao3downloader/actions/shared.py

@@ -147,12 +147,15 @@ def ao3_login(repo: Repository, fileops: FileOps, force: bool=False) -> None:
         login = False if input() == strings.PROMPT_NO else True
 
     if login:
+        savepassword = fileops.get_ini_value_boolean(strings.INI_PASSWORD_SAVE, True)
+
         username = fileops.setting(
             strings.AO3_PROMPT_USERNAME,
             strings.SETTING_USERNAME)
         password = fileops.setting(
             strings.AO3_PROMPT_PASSWORD,
-            strings.SETTING_PASSWORD)
+            strings.SETTING_PASSWORD,
+            savepassword)
 
         print(strings.AO3_INFO_LOGIN)
         try:

ao3downloader/actions/updatefics.py

@@ -10,8 +10,8 @@
 
 
 def action():
-    with Repository() as repo:
-        fileops = FileOps()
+    fileops = FileOps()
+    with Repository(fileops) as repo:
 
         folder = shared.update_folder(fileops)
         update_filetypes = shared.update_types(fileops)

ao3downloader/actions/updateseries.py

@@ -9,8 +9,8 @@
 
 
 def action():
-    with Repository() as repo:
-        fileops = FileOps()
+    fileops = FileOps()
+    with Repository(fileops) as repo:
 
         folder = shared.update_folder(fileops)
         update_filetypes = shared.update_types(fileops)

ao3downloader/fileio.py

@@ -1,5 +1,6 @@
 """File operations go here."""
 
+import configparser
 import datetime
 import json
 import os
@@ -12,6 +13,7 @@ def __init__(self):
         if not os.path.exists(strings.LOG_FOLDER_NAME): os.mkdir(strings.LOG_FOLDER_NAME)
         if not os.path.exists(strings.DOWNLOAD_FOLDER_NAME): os.mkdir(strings.DOWNLOAD_FOLDER_NAME)
         self.logfile = os.path.join(strings.LOG_FOLDER_NAME, strings.LOG_FILE_NAME)
+        self.inifile = strings.INI_FILE_NAME
         self.settingsfile = strings.SETTINGS_FILE_NAME
         self.downloadfolder = strings.DOWNLOAD_FOLDER_NAME
 
@@ -58,12 +60,13 @@ def get_settings_json(self) -> dict:
                 return {}
 
 
-    def setting(self, prompt: str, setting: str):
+    def setting(self, prompt: str, setting: str, save: bool = True):
         value = self.get_setting(setting)
         if value == '':
             print(prompt)
             value = input()
-            self.save_setting(setting, value)
+            if save:
+                self.save_setting(setting, value)
         return value
 
 
@@ -86,3 +89,15 @@ def file_exists(self, id: str, titles: dict[str, str], filetypes: list[str]) ->
             if not os.path.exists(file):
                 return False
         return True
+
+
+    def get_ini_value(self, key: str, fallback: str = None) -> str:
+        config = configparser.ConfigParser()
+        config.read(self.inifile)
+        return config.get(strings.INI_SECTION_NAME, key, fallback=fallback)
+
+
+    def get_ini_value_boolean(self, key: str, fallback: bool) -> bool:
+        config = configparser.ConfigParser()
+        config.read(self.inifile)
+        return config.getboolean(strings.INI_SECTION_NAME, key, fallback=fallback)

ao3downloader/repo.py

@@ -9,17 +9,17 @@
 from requests import codes
 
 from ao3downloader import exceptions, parse_soup, parse_text, strings
+from ao3downloader.fileio import FileOps
 
 
 class Repository:
 
-    # for reasons I don't fully understand, specifying the user agent makes requests faster
-    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit'
-            '/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36 [email protected]'}
+    headers = {'user-agent': 'ao3downloader [email protected]'}
 
 
-    def __init__(self) -> None:
+    def __init__(self, fileops: FileOps) -> None:
         self.session = requests.Session()
+        self.extra_wait = int(fileops.get_ini_value(strings.INI_WAIT_TIME, '0'))
 
 
     def __enter__(self):
@@ -63,13 +63,16 @@ def my_get(self, url: str) -> requests.Response:
                 pause_time = int(response.headers['retry-after'])
             except:
                 pause_time = 300 # default to 5 minutes in case there was a problem getting retry-after
+            if pause_time < 0: pause_time = 300 # also do 5 minutes if retry-after was found but is invalid
             now = datetime.datetime.now()
             later = now + datetime.timedelta(0, pause_time)
             print(strings.MESSAGE_TOO_MANY_REQUESTS.format(pause_time, now.strftime('%H:%M:%S'), later.strftime('%H:%M:%S')))
             sleep(pause_time)
             print(strings.MESSAGE_RESUMING)
             return self.my_get(url)
 
+        if self.extra_wait > 0: sleep(self.extra_wait)
+
         return response
 
 
@@ -79,7 +82,7 @@ def login(self, username: str, password: str):
         soup = self.get_soup(strings.AO3_LOGIN_URL)
         token = parse_soup.get_token(soup)
         payload = parse_text.get_payload(username, password, token)
-        response = self.session.post(strings.AO3_LOGIN_URL, data=payload)
+        response = self.session.post(strings.AO3_LOGIN_URL, data=payload, headers=self.headers)
         soup = BeautifulSoup(response.text, 'html.parser')
         if parse_soup.is_failed_login(soup):
             raise exceptions.LoginException(strings.ERROR_FAILED_LOGIN)

ao3downloader/strings.py

@@ -13,6 +13,11 @@
 TEMPLATE_FILE_NAME = 'template.html'
 VISUALIZATION_FILE_NAME = 'logvisualization{}.html'
 IGNORELIST_FILE_NAME = 'ignorelist.txt'
+INI_FILE_NAME = 'settings.ini'
+INI_SECTION_NAME = 'settings'
+
+INI_WAIT_TIME = 'ExtraWaitTime'
+INI_PASSWORD_SAVE = 'SavePassword'
 
 SETTING_USERNAME = 'username'
 SETTING_PASSWORD = 'password'

settings.ini

@@ -0,0 +1,14 @@
+[settings]
+
+# this is the number of seconds to wait between each hit to ao3 IN 
+# ADDITION to the required wait times when the rate limit is hit.
+# you may wish to set this above zero if you feel that you are
+# hitting the rate limit too often, or if you wish to avoid hitting
+# the rate limit for some reason (like if you are actively browsing
+# while the script is running, or you encounter rate-limit related bugs)
+ExtraWaitTime=0
+
+# if you set this to 'false' your password will not be saved in settings.
+# note that if you already saved your password on a previous run, it will
+# not be deleted. to fix this you can delete the 'settings.json' file
+SavePassword=true