Add new features for archiving (#24)

src/wxflow/executable.py

@@ -15,12 +15,12 @@ class Executable:
     --------
 
     >>> from wxflow.executable import Executable
-    >>> cmd = Executable('srun')  # Lets say we need to run command e.g. "srun"
-    >>> cmd.add_default_arg('my_exec.x')  # Lets say we need to run the executable "my_exec.x"
-    >>> cmd.add_default_arg('my_arg.yaml')  # Lets say we need to pass an argument to this executable e.g. "my_arg.yaml"
-    >>> cmd.add_default_env('OMP_NUM_THREADS', 4)  # Lets say we want to run w/ 4 threads in the environment
-    >>> cmd(output='stdout', error='stderr')  # Run the command and capture the stdout and stderr in files named similarly.
-
+    >>> cmd = Executable('srun')  # Let's say we need to run command e.g. "srun"
+    >>> arg_list = ['my_exec.x']  # Let's say we need to run the executable "my_exec.x"
+    >>> arg_list.append('my_arg.yaml')  # Let's say we need to pass an argument to this executable e.g. "my_arg.yaml"
+    >>> env = os.environ.copy(); env['OMP_NUM_THREADS'] = 4  # Let's say we want to run w/ 4 threads in the environment
+    >>> # Run the command with the arguments and environment and capture the stdout and stderr in files named similarly.
+    >>> cmd(*arg_list, env = env, output='stdout', error='stderr')
     `cmd` line above will translate to:
 
     $ export OMP_NUM_THREADS=4

src/wxflow/htar.py

@@ -54,7 +54,8 @@ def _htar(self, arg_list: list, silent: bool = False) -> str:
 
         return output
 
-    def create(self, tarball: str, fileset: Union[List, str], opts: Union[List, str] = "-P") -> str:
+    def create(self, tarball: str, fileset: Union[List, str],
+               dereference: bool = False, opts: Union[List, str] = "-P") -> str:
         """ Method to write an archive to HPSS
 
         Parameters
@@ -68,13 +69,19 @@ def create(self, tarball: str, fileset: Union[List, str], opts: Union[List, str]
         fileset : list | str
                 List containing filenames, patterns, or directories to archive
 
+        dereference : bool
+                Whether to dereference symbolic links (archive the pointed-to files instead).
+
         Returns
         -------
         output : str
                 Concatenated output and error of the htar command.
         """
         arg_list = ["-c"]
 
+        if dereference:
+            arg_list.append("-h")
+
         # Parse any htar options
         arg_list.extend(Htar._split_opts(opts))
 
@@ -90,7 +97,7 @@ def create(self, tarball: str, fileset: Union[List, str], opts: Union[List, str]
 
         return output
 
-    def cvf(self, tarball: str, fileset: Union[List, str]) -> str:
+    def cvf(self, tarball: str, fileset: Union[List, str], dereference: bool = False) -> str:
         """ Method to write an archive to HPSS verbosely (without options).
 
         Parameters
@@ -101,12 +108,15 @@ def cvf(self, tarball: str, fileset: Union[List, str]) -> str:
         fileset : list | str
                 List containing filenames, patterns, or directories to archive
 
+        dereference : bool
+                Whether to dereference symbolic links (archive the pointed-to files instead).
+
         Returns
         -------
         output : str
                 Concatenated output and error from the htar command
         """
-        output = self.create(tarball, fileset, opts="-v -P")
+        output = self.create(tarball, fileset, dereference=dereference, opts="-v -P")
 
         return output
 

src/wxflow/jinja.py

@@ -106,6 +106,7 @@ def get_set_env(self, loader: jinja2.BaseLoader, filters: Dict[str, callable] =
         to_YMD: convert a datetime object to a YYYYMMDD string
         to_julian: convert a datetime object to a julian day
         to_f90bool: convert a boolean to a fortran boolean
+        relpath: convert a full path to a relative path based on an input root_path
         getenv: read variable from environment if defined, else UNDEFINED
 
         Parameters
@@ -129,6 +130,7 @@ def get_set_env(self, loader: jinja2.BaseLoader, filters: Dict[str, callable] =
         env.filters["to_julian"] = lambda dt: to_julian(dt) if not isinstance(dt, SilentUndefined) else dt
         env.filters["to_f90bool"] = lambda bool: ".true." if bool else ".false."
         env.filters['getenv'] = lambda name, default='UNDEFINED': os.environ.get(name, default)
+        env.filters["relpath"] = lambda pathname, start=os.curdir: os.path.relpath(pathname, start)
 
         # Add any additional filters
         if filters is not None:

src/wxflow/yaml_file.py

@@ -177,4 +177,7 @@ def parse_j2yaml(path: str, data: Dict, searchpath: Union[str, List] = '/') -> D
         the dict configuration
     """
 
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"Input j2yaml file {path} does not exist!")
+
     return YAMLFile(data=Jinja(path, data, searchpath=searchpath).render)

tests/test_htar.py

@@ -40,10 +40,14 @@ def test_cvf_xvf_tell(tmp_path):
         f.touch()
         f.write_text("Some contents")
 
+    # Create a symlink, add it to tmp_files
+    os.symlink("a.txt", input_dir_path / "ln_a.txt")
+    in_tmp_files.append(input_dir_path / "ln_a.txt")
+
     test_tarball = test_path + "/test.tar"
 
     # Create the archive file
-    output = htar.cvf(test_tarball, in_tmp_files)
+    output = htar.cvf(test_tarball, in_tmp_files, dereference=True)
 
     assert "a.txt" in output
     assert hsi.exists(test_tarball)
@@ -52,6 +56,7 @@ def test_cvf_xvf_tell(tmp_path):
     output = htar.xvf(test_tarball, in_tmp_files)
 
     assert "a.txt" in output
+    assert "ln_a.txt" in output
 
     # List the contents of the test archive
     output = htar.tell(test_tarball)