Merge pull request #1392 from pyiron/fb

Misc FileBrowser features

pyiron_base/jobs/job/extension/files.py

@@ -1,6 +1,7 @@
 import os
 import posixpath
-from typing import List
+from typing import List, Optional
+from itertools import islice
 from pyiron_base.jobs.job.util import (
     _working_directory_list_files,
     _working_directory_read_file,
@@ -67,7 +68,7 @@ def list(self) -> List[str]:
     def _ipython_display_(self):
         path = self._job.working_directory + ":"
         files = [
-            "\t" + f
+            "\t" + str(f)
             for f in _working_directory_list_files(
                 working_directory=self._working_directory
             )
@@ -85,14 +86,12 @@ def tail(self, file: str, lines: int = 100):
         Raises:
             FileNotFoundError: if the given file does not exist
         """
-        print(
-            *_working_directory_read_file(
-                working_directory=self._working_directory, file_name=file, tail=lines
-            ),
-            sep="",
-        )
+        return self[file].tail(lines=lines)
 
     def __getitem__(self, item):
+        sub = os.path.join(self._working_directory, item)
+        if os.path.isdir(sub):
+            return FileBrowser(sub)
         if item in _working_directory_list_files(
             working_directory=self._working_directory,
             include_archive=False,
@@ -125,15 +124,36 @@ def __init__(self, path):
     def __str__(self):
         return self._path
 
-    def tail(self, lines: int = 100):
-        print(
-            *_working_directory_read_file(
-                working_directory=os.path.dirname(self._path),
-                file_name=os.path.basename(self._path),
-                tail=lines,
-            ),
-            sep="",
+    def _read(self, tail=None):
+        return _working_directory_read_file(
+            working_directory=os.path.dirname(str(self)),
+            file_name=os.path.basename(str(self)),
+            tail=tail,
         )
 
+    def __iter__(self):
+        return iter(self._read())
+
+    def list(self, lines: Optional[int] = None):
+        """
+        Return file content as list of lines.
+
+        Args:
+            lines (int): only return the first `lines` lines
+
+        Return:
+            list of str: file content
+        """
+        return list(islice(iter(self), lines))
+
+    def tail(self, lines: int = 100):
+        """
+        Print the last `lines` to stdout.
+
+        Args:
+            lines (int): number of output lines
+        """
+        print(*self._read(tail=lines), sep="")
+
     def __eq__(self, other):
         return self.__str__().__eq__(other)

tests/flex/test_executablecontainer.py

@@ -100,7 +100,7 @@ def test_job_files(self):
             self.assertTrue(file in dir(job.files))
         output_file_path = os.path.abspath(os.path.join(__file__, "..", "test_executablecontainer", "job_output_files_hdf5", "job_output_files", "error.out"))
         if os.name != "nt":
-            self.assertEqual(job.files.error_out, output_file_path)
+            self.assertEqual(str(job.files.error_out), output_file_path)
         else:
             self.assertEqual(job.files.error_out, output_file_path.replace("\\", "/"))
 

tests/job/test_genericJob.py

@@ -625,7 +625,7 @@ def test_generate_flux_execute_string(self):
         )
 
     def test_tail(self):
-        """job.tail should print the last lines of a file to stdout"""
+        """job.files.tail should print the last lines of a file to stdout"""
         job = self.project.load(self.project.get_job_ids()[0])
         job.decompress()
         content = ["Content", "More", "Lines"]
@@ -634,30 +634,64 @@ def test_tail(self):
 
         for i in range(len(content)):
             with self.subTest(i=i):
+                reference_str = os.linesep.join(content[-i-1:]) + os.linesep
                 with contextlib.redirect_stdout(io.StringIO(newline=os.linesep)) as f:
                     job.files.tail("test_file", lines=i+1)
-                reference_str = os.linesep.join(content[-i-1:]) + os.linesep
                 self.assertEqual(f.getvalue().replace('\r', ''), reference_str.replace('\r', ''),
                                  "tail read incorrect lines from output file when job uncompressed!")
                 with contextlib.redirect_stdout(io.StringIO(newline=os.linesep)) as f:
                     job.files.test_file.tail(lines=i+1)
-                reference_str = os.linesep.join(content[-i-1:]) + os.linesep
                 self.assertEqual(f.getvalue().replace('\r', ''), reference_str.replace('\r', ''),
                                  "tail read incorrect lines from output file when job uncompressed!")
 
         job.compress()
         for i in range(len(content)):
+            reference_str = os.linesep.join(content[-i-1:]) + os.linesep
             with contextlib.redirect_stdout(io.StringIO()) as f:
                 job.files.tail("test_file", lines=i+1)
-            reference_str = os.linesep.join(content[-i-1:]) + os.linesep
             self.assertEqual(f.getvalue().replace('\r', ''), reference_str.replace('\r', ''),
                              "tail read incorrect lines from output file when job compressed!")
             with contextlib.redirect_stdout(io.StringIO()) as f:
                 job.files.test_file.tail(lines=i+1)
-            reference_str = os.linesep.join(content[-i-1:]) + os.linesep
             self.assertEqual(f.getvalue().replace('\r', ''), reference_str.replace('\r', ''),
                              "tail read incorrect lines from output file when job compressed!")
 
+    def test_file_lines(self):
+        """job.files....list/iter should return the (first) lines of a file"""
+        job = self.project.load(self.project.get_job_ids()[0])
+        job.decompress()
+        content = ["Content\n", "More\n", "Lines"]
+        with open(os.path.join(job.working_directory, "test_file"), "w") as f:
+            f.write("".join(content))
+
+        for count, (l, l_ref) in enumerate(zip(job.files.test_file, content)):
+            self.assertEqual(l.replace("\r", ""), l_ref,
+                             "Iterating over file doesn't return same line as was written!")
+        self.assertEqual(count + 1, len(content), "Iterating over file skipped some lines!")
+
+        for i in range(len(content)):
+            with self.subTest(i=i):
+                reference_str = "".join(content[:i+1])
+                test_str = ''.join(job.files.test_file.list(lines=i+1))
+                self.assertEqual(test_str.replace('\r', ''), reference_str.replace('\r', ''),
+                                 "list reads incorrect lines from output file when job uncompressed!")
+
+        job.compress()
+        for i in range(len(content)):
+            with self.subTest(i=i):
+                reference_str = "".join(content[:i+1])
+                test_str = ''.join(job.files.test_file.list(lines=i+1))
+                self.assertEqual(test_str.replace('\r', ''), reference_str.replace('\r', ''),
+                                 "list reads incorrect lines from output file when job uncompressed!")
+
+    def test_files_subdir(self):
+        """It should be possible to access sub directories of job folders with FileBrowser."""
+
+        job = self.project.load(self.project.get_job_ids()[0])
+        os.makedirs(job.working_directory + "/subdir")
+        self.assertTrue("subdir" in job.files.list(), "FileBrowser doesn't list sub directories.")
+        self.assertTrue(isinstance(job.files["subdir"], type(job.files)),
+                        "Accessing a sub directory via FileBrowser does not return another FileBrowser!")
 
 if __name__ == "__main__":
     unittest.main()