From 4b2c06bb99fa78511570770c623c6c0cab13d1e5 Mon Sep 17 00:00:00 2001 From: Daniel Sola <40698988+dansola@users.noreply.github.com> Date: Fri, 14 Jun 2024 10:24:16 -0700 Subject: [PATCH] Add functionality for .flyteignore file (#2479) * Add functionality for .flyteignore file Signed-off-by: Daniel Sola * copy docker ignore for flyteignore --------- Signed-off-by: Daniel Sola --- flytekit/tools/fast_registration.py | 4 +-- flytekit/tools/ignore.py | 21 ++++++++++++++ tests/flytekit/unit/tools/test_ignore.py | 36 ++++++++++++++++++++++-- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/flytekit/tools/fast_registration.py b/flytekit/tools/fast_registration.py index c67873c9d8..6108048533 100644 --- a/flytekit/tools/fast_registration.py +++ b/flytekit/tools/fast_registration.py @@ -15,7 +15,7 @@ from flytekit.core.context_manager import FlyteContextManager from flytekit.core.utils import timeit -from flytekit.tools.ignore import DockerIgnore, GitIgnore, Ignore, IgnoreGroup, StandardIgnore +from flytekit.tools.ignore import DockerIgnore, FlyteIgnore, GitIgnore, Ignore, IgnoreGroup, StandardIgnore from flytekit.tools.script_mode import tar_strip_file_attributes FAST_PREFIX = "fast" @@ -46,7 +46,7 @@ def fast_package( :param bool deref_symlinks: Enables dereferencing symlinks when packaging directory :return os.PathLike: """ - default_ignores = [GitIgnore, DockerIgnore, StandardIgnore] + default_ignores = [GitIgnore, DockerIgnore, StandardIgnore, FlyteIgnore] if options is not None: if options.keep_default_ignores: ignores = options.ignores + default_ignores diff --git a/flytekit/tools/ignore.py b/flytekit/tools/ignore.py index 4a427c2734..e41daf0904 100644 --- a/flytekit/tools/ignore.py +++ b/flytekit/tools/ignore.py @@ -87,6 +87,27 @@ def _is_ignored(self, path: str) -> bool: return self.pm.matches(path) +class FlyteIgnore(Ignore): + """Uses a .flyteignore file to determine ignored files.""" + + def __init__(self, root: Path): + super().__init__(root) + self.pm = self._parse() + + def _parse(self) -> PatternMatcher: + patterns = [] + flyteignore = os.path.join(self.root, ".flyteignore") + if os.path.isfile(flyteignore): + with open(flyteignore, "r") as f: + patterns = [l.strip() for l in f.readlines() if l and not l.startswith("#")] + else: + logger.info(f"No .flyteignore found in {self.root}, not applying any filters") + return PatternMatcher(patterns) + + def _is_ignored(self, path: str) -> bool: + return self.pm.matches(path) + + class StandardIgnore(Ignore): """Retains the standard ignore functionality that previously existed. Could in theory by fed with custom ignore patterns from cli.""" diff --git a/tests/flytekit/unit/tools/test_ignore.py b/tests/flytekit/unit/tools/test_ignore.py index 3b24eeb8a0..614269fc26 100644 --- a/tests/flytekit/unit/tools/test_ignore.py +++ b/tests/flytekit/unit/tools/test_ignore.py @@ -8,7 +8,7 @@ import pytest from docker.utils.build import PatternMatcher -from flytekit.tools.ignore import DockerIgnore, GitIgnore, IgnoreGroup, StandardIgnore +from flytekit.tools.ignore import DockerIgnore, GitIgnore, IgnoreGroup, StandardIgnore, FlyteIgnore def make_tree(root: Path, tree: Dict): @@ -102,6 +102,19 @@ def all_ignore(tmp_path): return tmp_path +@pytest.fixture +def simple_flyteignore(tmp_path): + tree = { + "sub": {"some.bar": ""}, + "test.foo": "", + "keep.foo": "", + ".flyteignore": "\n".join(["*.foo", "!keep.foo", "# A comment", "sub"]), + } + + make_tree(tmp_path, tree) + return tmp_path + + def test_simple_gitignore(simple_gitignore): gitignore = GitIgnore(simple_gitignore) assert gitignore.is_ignored(str(simple_gitignore / "test.foo")) @@ -219,7 +232,7 @@ def test_all_ignore(all_ignore): def test_all_ignore_tar_filter(all_ignore): """Test tar_filter method of all ignores grouped together""" - ignore = IgnoreGroup(all_ignore, [GitIgnore, DockerIgnore, StandardIgnore]) + ignore = IgnoreGroup(all_ignore, [GitIgnore, DockerIgnore, StandardIgnore, FlyteIgnore]) assert ignore.tar_filter(TarInfo(name="sub")).name == "sub" assert ignore.tar_filter(TarInfo(name="sub/some.bar")).name == "sub/some.bar" assert not ignore.tar_filter(TarInfo(name="sub/__pycache__/")) @@ -232,4 +245,23 @@ def test_all_ignore_tar_filter(all_ignore): assert ignore.tar_filter(TarInfo(name="keep.foo")).name == "keep.foo" assert ignore.tar_filter(TarInfo(name=".gitignore")).name == ".gitignore" assert ignore.tar_filter(TarInfo(name=".dockerignore")).name == ".dockerignore" + assert ignore.tar_filter(TarInfo(name=".flyteignore")).name == ".flyteignore" assert not ignore.tar_filter(TarInfo(name=".git")) + + +def test_flyteignore_parse(simple_flyteignore): + """Test .flyteignore file parsing""" + flyteignore = FlyteIgnore(simple_flyteignore) + assert flyteignore.pm.matches("whatever.foo") + assert not flyteignore.pm.matches("keep.foo") + assert flyteignore.pm.matches("sub") + assert flyteignore.pm.matches("sub/stuff.txt") + + +def test_simple_flyteignore(simple_flyteignore): + flyteignore = FlyteIgnore(simple_flyteignore) + assert flyteignore.is_ignored(str(simple_flyteignore / "test.foo")) + assert flyteignore.is_ignored(str(simple_flyteignore / "sub")) + assert flyteignore.is_ignored(str(simple_flyteignore / "sub" / "some.bar")) + assert not flyteignore.is_ignored(str(simple_flyteignore / "keep.foo")) + assert not flyteignore.is_ignored(str(simple_flyteignore / ".flyteignore"))