Periodic Rescans Added (New Feature)

.github/workflows/build.yml

@@ -35,28 +35,10 @@ jobs:
             echo "valid_branch_name=true" >> $GITHUB_OUTPUT
           fi
 
-  lint:
+  unit-tests:
     needs: validate-branch-name
     if: needs.validate-branch-name.outputs.valid_branch_name == 'true'
     runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10.13'
-      - name: Install linting dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pylint
-      - name: Run lint checks
-        run: |
-          echo "here we'll lint" 
-          # pylint my_project
-
-  unit-tests:
-    needs: lint
-    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python

.pre-commit-config.yaml

@@ -3,9 +3,8 @@ repos:
     hooks:
       - id: black
         name: black
-        entry: venv/bin/black
+        entry: venv/bin/black --config pyproject.toml
         language: system
         types: [python]
 
 
-

.pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-log_cli = true
+# log_cli = true
 addopts = -q --tb=short -s
 log_cli_level = INFO
 log_cli_format = %(asctime)s - %(levelname)s - %(name)s - %(message)s

README.md

@@ -55,47 +55,66 @@ services:
     container_name: decluttarr
     restart: always
     environment:
-      - TZ=Europe/Zurich
-      - PUID=1000
-      - PGID=1000
-      ## General
-      - LOG_LEVEL=INFO
-      #- TEST_RUN=True
-      #- SSL_VERIFICATION=False
-      ## Features 
-      - REMOVE_TIMER=10
-      - REMOVE_FAILED=True
-      - REMOVE_FAILED_IMPORTS=True
-      - REMOVE_METADATA_MISSING=True
-      - REMOVE_MISSING_FILES=True     
-      - REMOVE_ORPHANS=True
-      - REMOVE_SLOW=True
-      - REMOVE_STALLED=True
-      - REMOVE_UNMONITORED=True
-      - MIN_DOWNLOAD_SPEED=100
-      - PERMITTED_ATTEMPTS=3
-      - NO_STALLED_REMOVAL_QBIT_TAG=Don't Kill
-      - IGNORE_PRIVATE_TRACKERS=True
-      - FAILED_IMPORT_MESSAGE_PATTERNS=["Not an upgrade for existing", "Not a Custom Format upgrade for existing"]
-      ## Radarr
-      - RADARR_URL=http://radarr:7878
-      - RADARR_KEY=$RADARR_API_KEY
-      ## Sonarr
-      - SONARR_URL=http://sonarr:8989
-      - SONARR_KEY=$SONARR_API_KEY
-      ## Lidarr
-      - LIDARR_URL=http://lidarr:8686
-      - LIDARR_KEY=$LIDARR_API_KEY
-      ## Readarr
-      - READARR_URL=http://readarr:8787
-      - READARR_KEY=$READARR_API_KEY
-      ## Whisparr
-      - WHISPARR_URL=http://whisparr:6969
-      - WHISPARR_KEY=$WHISPARR_API_KEY
-      ## qBittorrent
-      - QBITTORRENT_URL=http://qbittorrent:8080
-      #- QBITTORRENT_USERNAME=Your name
-      #- QBITTORRENT_PASSWORD=Your password
+     TZ=Europe/Zurich
+     PUID=1000
+     PGID=1000
+
+     ## General
+     # TEST_RUN=True
+     # SSL_VERIFICATION=False
+     LOG_LEVEL: INFO
+
+     ## Features 
+     REMOVE_TIMER: 10
+     REMOVE_FAILED: True
+     REMOVE_FAILED_IMPORTS: True
+     REMOVE_METADATA_MISSING: True
+     REMOVE_MISSING_FILES: True
+     REMOVE_ORPHANS: True
+     REMOVE_SLOW: True
+     REMOVE_STALLED: True
+     REMOVE_UNMONITORED: True
+     RUN_PERIODIC_RESCANS: '
+        {
+          "SONARR": {"MISSING": true, "CUTOFF_UNMET": true, "MAX_CONCURRENT_SCANS": 3, "MIN_DAYS_BEFORE_RESCAN": 7},
+          "RADARR": {"MISSING": true, "CUTOFF_UNMET": true, "MAX_CONCURRENT_SCANS": 3, "MIN_DAYS_BEFORE_RESCAN": 7}
+        }'
+
+     # Feature Settings
+     PERMITTED_ATTEMPTS: 3
+     NO_STALLED_REMOVAL_QBIT_TAG: Don't Kill
+     REMOVE_SLOW: True
+     MIN_DOWNLOAD_SPEED: 100     
+     FAILED_IMPORT_MESSAGE_PATTERNS: '
+        [
+          "Not a Custom Format upgrade for existing", 
+          "Not an upgrade for existing"
+        ]'
+
+     ## Radarr
+     RADARR_URL: http://radarr:7878
+     RADARR_KEY: $RADARR_API_KEY
+
+     ## Sonarr
+     SONARR_URL: http://sonarr:8989
+     SONARR_KEY: $SONARR_API_KEY
+
+     ## Lidarr
+     LIDARR_URL=http://lidarr:8686
+     LIDARR_KEY=$LIDARR_API_KEY
+
+     ## Readarr
+     READARR_URL=http://readarr:8787
+     READARR_KEY=$READARR_API_KEY
+
+     ## Whisparr
+     WHISPARR_URL=http://whisparr:6969
+     WHISPARR_KEY=$WHISPARR_API_KEY
+
+     ## qBitorrent
+     QBITTORRENT_URL: http://qbittorrent:8080
+     # QBITTORRENT_USERNAME=Your name
+     # QBITTORRENT_PASSWORD=Your password
 ```
 3) Run `docker-compose up -d` in the directory where the file is located to create the docker container
 Note: Always pull the "**latest**" version. The "dev" version is for testing only, and should only be pulled when contributing code or supporting with bug fixes
@@ -212,6 +231,26 @@ Steers which type of cleaning is applied to the downloads queue
 - Permissible Values: True, False
 - Is Mandatory: No (Defaults to False)
 
+**RUN_PERIODIC_RESCANS**
+- Steers whether searches are automatically triggered for items that are missing or have not yet met the cutoff
+- Note: Only supports Radarr/Sonarr currently (Lidarr depending on: https://github.com/Lidarr/Lidarr/pull/5084 / Readarr Depending on: https://github.com/Readarr/Readarr/pull/3724)
+- Type: Dictionaire
+- Is Mandatory: No (Defaults to no searches being triggered automatically)
+- "SONARR"/"RADARR" turns on the automatic searches for the respective instances
+- "MISSING"/"CUTOFF_UNMET" turns on the automatic search for those wanted items (defaults to True)
+- "MAX_CONCURRENT_SCANS" specifies the maximum number of items to be searched in each scan. This value dictates how many items are processed per search operation, which occurs according to the interval set by the REMOVE_TIMER.
+- Note: The limit is per wanted list. Thus if both Radarr & Sonarr are set up for automatic searches, both for missing and cutoff unmet items, the actual count may be four times the MAX_CONCURRENT_SCANS
+- "MIN_DAYS_BEFORE_RESCAN" steers the days that need to pass before an item is considered again for a scan
+- Note: RUN_PERIODIC_RESCANS will always search those items that haven been searched for longest
+
+```
+     RUN_PERIODIC_RESCANS: '
+        {
+          "SONARR": {"MISSING": true, "CUTOFF_UNMET": true, "MAX_CONCURRENT_SCANS": 3, "MIN_DAYS_BEFORE_RESCAN": 7},
+          "RADARR": {"MISSING": true, "CUTOFF_UNMET": true, "MAX_CONCURRENT_SCANS": 3, "MIN_DAYS_BEFORE_RESCAN": 7}
+        }'
+```
+
 **MIN_DOWNLOAD_SPEED**
 - Sets the minimum download speed for active downloads
 - If the increase in the downloaded file size of a download is less than this value between two consecutive checks, the download is considered slow and is removed if happening more ofthen than the permitted attempts

config/config.conf-Example

@@ -12,6 +12,9 @@ REMOVE_ORPHANS              = True
 REMOVE_SLOW                 = True
 REMOVE_STALLED              = True
 REMOVE_UNMONITORED          = True
+RUN_PERIODIC_RESCANS        = {"SONARR": {"MISSING": true, CUTOFF_UNMET": true, "MAX_CONCURRENT_SCANS": 3, "MIN_DAYS_BEFORE_RESCAN": 7}, "RADARR": {"MISSING": true, "CUTOFF_UNMET": true, "MAX_CONCURRENT_SCANS": 3, "MIN_DAYS_BEFORE_RESCAN": 7}}
+
+[feature_settings]
 MIN_DOWNLOAD_SPEED          = 100
 PERMITTED_ATTEMPTS          = 3
 NO_STALLED_REMOVAL_QBIT_TAG = Don't Kill

config/definitions.py

@@ -12,17 +12,20 @@
 REMOVE_FAILED                   = get_config_value('REMOVE_FAILED',                 'features',     False,  bool,   False)
 REMOVE_FAILED_IMPORTS           = get_config_value('REMOVE_FAILED_IMPORTS' ,        'features',     False,  bool,   False)
 REMOVE_METADATA_MISSING         = get_config_value('REMOVE_METADATA_MISSING',       'features',     False,  bool,   False)
-REMOVE_MISSING_FILES            = get_config_value('REMOVE_MISSING_FILES' ,         'features',     False,  bool,   False)
-REMOVE_NO_FORMAT_UPGRADE        = get_config_value('REMOVE_NO_FORMAT_UPGRADE' ,     'features',     False,  bool,   False) # OUTDATED - WILL RETURN WARNING
-REMOVE_ORPHANS                  = get_config_value('REMOVE_ORPHANS' ,               'features',     False,  bool,   False)
-REMOVE_SLOW                     = get_config_value('REMOVE_SLOW' ,                  'features',     False,  bool,   False)
+REMOVE_MISSING_FILES            = get_config_value('REMOVE_MISSING_FILES',          'features',     False,  bool,   False)
+REMOVE_NO_FORMAT_UPGRADE        = get_config_value('REMOVE_NO_FORMAT_UPGRADE',      'features',     False,  bool,   False) # OUTDATED - WILL RETURN WARNING
+REMOVE_ORPHANS                  = get_config_value('REMOVE_ORPHANS',                'features',     False,  bool,   False)
+REMOVE_SLOW                     = get_config_value('REMOVE_SLOW',                   'features',     False,  bool,   False)
 REMOVE_STALLED                  = get_config_value('REMOVE_STALLED',                'features',     False,  bool,   False)
-REMOVE_UNMONITORED              = get_config_value('REMOVE_UNMONITORED' ,           'features',     False,  bool,   False)
-MIN_DOWNLOAD_SPEED              = get_config_value('MIN_DOWNLOAD_SPEED',            'features',     False,  int,    0)
-PERMITTED_ATTEMPTS              = get_config_value('PERMITTED_ATTEMPTS',            'features',     False,  int,    3)
-NO_STALLED_REMOVAL_QBIT_TAG     = get_config_value('NO_STALLED_REMOVAL_QBIT_TAG',   'features',     False,  str,   'Don\'t Kill')
-IGNORE_PRIVATE_TRACKERS         = get_config_value('IGNORE_PRIVATE_TRACKERS',       'features',     False,  bool,   True)
-FAILED_IMPORT_MESSAGE_PATTERNS  = get_config_value('FAILED_IMPORT_MESSAGE_PATTERNS','features',     False,  list,   [])
+REMOVE_UNMONITORED              = get_config_value('REMOVE_UNMONITORED',            'features',     False,  bool,   False)
+RUN_PERIODIC_RESCANS            = get_config_value('RUN_PERIODIC_RESCANS',          'features',     False,  dict,   {})
+
+# Feature Settings
+MIN_DOWNLOAD_SPEED              = get_config_value('MIN_DOWNLOAD_SPEED',            'feature_settings',     False,  int,    0)
+PERMITTED_ATTEMPTS              = get_config_value('PERMITTED_ATTEMPTS',            'feature_settings',     False,  int,    3)
+NO_STALLED_REMOVAL_QBIT_TAG     = get_config_value('NO_STALLED_REMOVAL_QBIT_TAG',   'feature_settings',     False,  str,   'Don\'t Kill')
+IGNORE_PRIVATE_TRACKERS         = get_config_value('IGNORE_PRIVATE_TRACKERS',       'feature_settings',     False,  bool,   True)
+FAILED_IMPORT_MESSAGE_PATTERNS  = get_config_value('FAILED_IMPORT_MESSAGE_PATTERNS','feature_settings',     False,  list,   [])
 
 # Radarr
 RADARR_URL                      = get_config_value('RADARR_URL',                    'radarr',       False,  str)
@@ -60,6 +63,41 @@
     print(f'[ ERROR ]: No Radarr/Sonarr/Lidarr/Readarr/Whisparr URLs specified (nothing to monitor)')
     exit()
 
+
+#### Validate rescan settings
+PERIODIC_RESCANS = get_config_value("PERIODIC_RESCANS", "features", False, dict, {})
+
+rescan_supported_apps = ["SONARR", "RADARR"]
+rescan_default_values = {
+    "MISSING": (True, bool),
+    "CUTOFF_UNMET": (True, bool),
+    "MAX_CONCURRENT_SCANS": (3, int),
+    "MIN_DAYS_BEFORE_RESCAN": (7, int),
+}
+
+
+# Remove rescan apps that are not supported
+for key in list(RUN_PERIODIC_RESCANS.keys()):
+    if key not in rescan_supported_apps:
+        print(f"[ WARNING ]: Removed '{key}' from RUN_PERIODIC_RESCANS since only {rescan_supported_apps} are supported.")
+        RUN_PERIODIC_RESCANS.pop(key)
+
+# Ensure SONARR and RADARR have the required parameters with default values if they are present
+for app in rescan_supported_apps:
+    if app in RUN_PERIODIC_RESCANS:
+        for param, (default, expected_type) in rescan_default_values.items():
+            if param not in RUN_PERIODIC_RESCANS[app]:
+                print(f"[ INFO ]: Adding missing parameter '{param}' to '{app}' with default value '{default}'.")
+                RUN_PERIODIC_RESCANS[app][param] = default
+            else:
+                # Check the type and correct if necessary
+                current_value = RUN_PERIODIC_RESCANS[app][param]
+                if not isinstance(current_value, expected_type):
+                    print(
+                        f"[ INFO ]: Parameter '{param}' for '{app}' must be of type {expected_type.__name__} and found value '{current_value}' (type '{type(current_value).__name__}'). Defaulting to '{default}'."
+                    )
+                    RUN_PERIODIC_RESCANS[app][param] = default
+
 ########### Enrich setting variables
 if RADARR_URL:      RADARR_URL =        RADARR_URL.rstrip('/')      + '/api/v3'
 if SONARR_URL:      SONARR_URL =        SONARR_URL.rstrip('/')      + '/api/v3'
@@ -68,8 +106,14 @@
 if WHISPARR_URL:    WHISPARR_URL =      WHISPARR_URL.rstrip('/')    + '/api/v3'
 if QBITTORRENT_URL: QBITTORRENT_URL =   QBITTORRENT_URL.rstrip('/') + '/api/v2'
 
-RADARR_MIN_VERSION          = '5.3.6.8608'
-SONARR_MIN_VERSION          = '4.0.1.1131'
+
+RADARR_MIN_VERSION = "5.3.6.8608"
+if "RADARR" in PERIODIC_RESCANS:
+    RADARR_MIN_VERSION = "5.10.3.9171"
+
+SONARR_MIN_VERSION = "4.0.1.1131"
+if "SONARR" in PERIODIC_RESCANS:
+    SONARR_MIN_VERSION = "4.0.9.2332"
 LIDARR_MIN_VERSION          = None
 READARR_MIN_VERSION         = None
 WHISPARR_MIN_VERSION        = '2.0.0.548'
@@ -82,4 +126,3 @@
 for var_name in dir():
     if var_name.isupper():
         settingsDict[var_name] = locals()[var_name]
-

config/parser.py

@@ -22,9 +22,14 @@ def config_section_map(section):
     options = config.options(section)
     for option in options:
         try:
-            dict1[option] = config.get(section, option)
-        except:
-            print("exception on %s!" % option)
+            value = config.get(section, option)
+            # Attempt to parse JSON for dictionary-like values
+            try:
+                dict1[option] = json.loads(value)
+            except json.JSONDecodeError:
+                dict1[option] = value
+        except Exception as e:
+            print(f"Exception on {option}: {e}")
             dict1[option] = None
     return dict1
 
@@ -38,44 +43,33 @@ def get_config_value(key, config_section, is_mandatory, datatype, default_value=
     if IS_IN_DOCKER:
         config_value = os.environ.get(key)
         if config_value is not None:
-            # print(f'The value retrieved for [{config_section}]: {key} is "{config_value}"')
             config_value = config_value
-            # return config_value
         elif is_mandatory:
             print(f"[ ERROR ]: Variable not specified in Docker environment: {key}")
             sys.exit(0)
         else:
-            # return default_value
-            # print(f'The default value used for [{config_section}]: {key} is "{default_value}" (data type: {type(default_value).__name__})')
             config_value = default_value
-
     else:
         try:
             config_value = config_section_map(config_section).get(key)
         except configparser.NoSectionError:
             config_value = None
         if config_value is not None:
-            # print(f'The value retrieved for [{config_section}]: {key} is "{config_value}"')
             config_value = config_value
-            # return config_value
         elif is_mandatory:
             print(
                 f"[ ERROR ]: Mandatory variable not specified in config file, section [{config_section}]: {key} (data type: {datatype.__name__})"
             )
             sys.exit(0)
         else:
-            # return default_value
-            # print(f'The default value used for [{config_section}]: {key} is "{default_value}" (data type: {type(default_value).__name__})')
             config_value = default_value
 
     # Apply data type
     try:
         if datatype == bool:
             config_value = eval(str(config_value).capitalize())
-        elif datatype == list:
-            if (
-                type(config_value) != list
-            ):  # Default value is already a list, doesn't need to be pushed through json.loads
+        elif datatype == list or datatype == dict:
+            if not isinstance(config_value, datatype):
                 config_value = json.loads(config_value)
         elif config_value is not None:
             config_value = cast(config_value, datatype)