add logic to make copies either required or optional

.vscode/settings.json

@@ -4,5 +4,5 @@
     "python.testing.autoTestDiscoverOnSaveEnabled": true,
     "python.testing.promptToConfigure": true,
     "python.testing.pytestArgs": ["-v", "--color=yes", "--no-cov"],
-    "python.testing.cwd": "./tests",
-}
+    "python.testing.cwd": "./tests"
+}

src/wxflow/file_utils.py

@@ -1,3 +1,4 @@
+import os
 from logging import getLogger
 
 from .fsutils import cp, mkdir
@@ -17,8 +18,10 @@ class FileHandler:
 
     NOTE
     ----
-    "action" can be one of mkdir", "copy", etc.
+    "action" can be one of "mkdir", "copy", "copy_req", "copy_opt", etc.
     Corresponding "act" would be ['dir1', 'dir2'], [['src1', 'dest1'], ['src2', 'dest2']]
+    "copy_req" will raise an error if the source file does not exist
+    "copy_opt" will not raise an error if the source file does not exist
 
     Attributes
     ----------
@@ -35,15 +38,25 @@ def sync(self):
         Method to execute bulk actions on files described in the configuration
         """
         sync_factory = {
-            'copy': self._copy_files,
+            'copy': self.copy_req,
+            'copy_req': self.copy_req,
+            'copy_opt': self.copy_opt,
             'mkdir': self._make_dirs,
         }
         # loop through the configuration keys
         for action, files in self.config.items():
             sync_factory[action](files)
 
     @staticmethod
-    def _copy_files(filelist):
+    def copy_req(filelist):
+        FileHandler._copy_files(filelist, required=True)
+
+    @staticmethod
+    def copy_opt(filelist):
+        FileHandler._copy_files(filelist, required=False)
+
+    @staticmethod
+    def _copy_files(filelist, required=True):
         """Function to copy all files specified in the list
 
         `filelist` should be in the form:
@@ -53,15 +66,28 @@ def _copy_files(filelist):
         ----------
         filelist : list
                 List of lists of [src, dest]
+        required : bool, optional
+                Flag to indicate if the src file is required to exist. Default is True
         """
         for sublist in filelist:
             if len(sublist) != 2:
-                raise Exception(
+                raise IndexError(
                     f"List must be of the form ['src', 'dest'], not {sublist}")
             src = sublist[0]
             dest = sublist[1]
-            cp(src, dest)
-            logger.info(f'Copied {src} to {dest}')
+            if os.path.exists(src):
+                try:
+                    cp(src, dest)
+                    logger.info(f'Copied {src} to {dest}')
+                except Exception as ee:
+                    logger.exception(f"Error copying {src} to {dest}")
+                    raise ee(f"Error copying {src} to {dest}")
+            else:
+                if required:
+                    logger.exception(f"Source file '{src}' does not exist and is required, ABORT!")
+                    raise FileNotFoundError(f"Source file '{src}' does not exist")
+                else:
+                    logger.warning(f"Source file '{src}' does not exist, skipping!")
 
     @staticmethod
     def _make_dirs(dirlist):
@@ -73,5 +99,9 @@ def _make_dirs(dirlist):
                 List of directories to create
         """
         for dd in dirlist:
-            mkdir(dd)
-            logger.info(f'Created {dd}')
+            try:
+                mkdir(dd)
+                logger.info(f'Created {dd}')
+            except Exception as ee:
+                logger.exception(f"Error creating directory {dd}")
+                raise ee(f"Error creating directory {dd}")

tests/test_file_utils.py

@@ -65,3 +65,18 @@ def test_copy(tmp_path):
     # Check if files were indeed copied
     for ff in dest_files:
         assert os.path.isfile(ff)
+
+    # Create a config object for copying optional non-existent files (c.txt does not exist)
+    copy_list.append([input_dir_path / 'c.txt', output_dir_path / 'c.txt'])
+    config = {'copy_opt': copy_list}
+
+    # Copy input files to output files
+    FileHandler(config).sync()
+
+    # Create a config object for copying required non-existent files (c.txt does not exist)
+    config = {'copy_req': copy_list}
+    try:
+        FileHandler(config).sync()
+    except FileNotFoundError as e:
+        c_file = input_dir_path / 'c.txt'
+        assert f"Source file '{c_file}' does not exist" in str(e)