Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove upper version bound from fsspec #2143

Merged
merged 10 commits into from
Feb 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions flytekit/types/directory/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

from flytekit.core.context_manager import FlyteContext, FlyteContextManager
from flytekit.core.type_engine import TypeEngine, TypeTransformer, get_batch_size
from flytekit.exceptions.user import FlyteAssertion
from flytekit.models import types as _type_models
from flytekit.models.core import types as _core_types
from flytekit.models.literals import Blob, BlobMetadata, Literal, Scalar
Expand Down Expand Up @@ -403,11 +404,7 @@ def to_literal(
if not isinstance(python_val.remote_directory, (pathlib.Path, str)) and (
python_val.remote_directory is False
or ctx.file_access.is_remote(source_path)
or ctx.execution_state.mode
in {
ctx.execution_state.Mode.LOCAL_WORKFLOW_EXECUTION,
ctx.execution_state.Mode.LOCAL_TASK_EXECUTION,
}
or ctx.execution_state.is_local_execution()
):
should_upload = False

Expand All @@ -431,6 +428,8 @@ def to_literal(
if should_upload:
if remote_directory is None:
remote_directory = ctx.file_access.get_random_remote_directory()
if not pathlib.Path(source_path).is_dir():
raise FlyteAssertion("Expected a directory. {} is not a directory".format(source_path))
ctx.file_access.put_data(source_path, remote_directory, is_multipart=True, batch_size=batch_size)
return Literal(scalar=Scalar(blob=Blob(metadata=meta, uri=remote_directory)))

Expand Down
8 changes: 4 additions & 4 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ readme = { file = "README.md", content-type = "text/markdown" }
requires-python = ">=3.8,<3.13"
dependencies = [
# Please maintain an alphabetical order in the following list
"adlfs>=2023.3.0,<=2023.9.2",
"adlfs>=2023.3.0",
"click>=6.6,<9.0",
"cloudpickle>=2.0.0",
"cookiecutter>=1.7.3",
Expand All @@ -22,8 +22,8 @@ dependencies = [
"docker>=4.0.0,<7.0.0",
"docstring-parser>=0.9.0",
"flyteidl>1.10.6",
"fsspec>=2023.3.0,<=2023.9.2",
"gcsfs>=2023.3.0,<=2023.9.2",
"fsspec>=2023.3.0",
"gcsfs>=2023.3.0",
"googleapis-common-protos>=1.57",
"grpcio",
"grpcio-status",
Expand All @@ -42,7 +42,7 @@ dependencies = [
"requests>=2.18.4,<3.0.0",
"rich",
"rich_click",
"s3fs>=2023.3.0,<=2023.9.2",
"s3fs>=2023.3.0",
"statsd>=3.0.0,<4.0.0",
"typing_extensions",
"urllib3>=1.22,<2.0.0",
Expand Down
2 changes: 1 addition & 1 deletion tests/flytekit/unit/core/test_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ def test_crawl_local_non_nt(source_folder):
assert set(files) == expected

# Test crawling a single file
fd = FlyteDirectory(path=os.path.join(source_folder, "original.txt"))
fd = FlyteDirectory(path=os.path.join(source_folder, "original1.txt"))
res = fd.crawl()
files = [os.path.join(x, y) for x, y in res]
assert len(files) == 0
Expand Down
4 changes: 1 addition & 3 deletions tests/flytekit/unit/core/test_flyte_directory.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,7 @@ def test_transformer_to_literal_local():
fs = FileAccessProvider(local_sandbox_dir=random_dir, raw_output_prefix=os.path.join(random_dir, "raw"))
ctx = context_manager.FlyteContext.current_context()
with context_manager.FlyteContextManager.with_context(ctx.with_file_access(fs)) as ctx:
# Use a separate directory that we know won't be the same as anything generated by flytekit itself, lest we
# accidentally try to cp -R /some/folder /some/folder/sub which causes exceptions obviously.
p = "/tmp/flyte/test_fd_transformer"
p = tempfile.mkdtemp(prefix="temp_example_")

# Create an empty directory and call to literal on it
if os.path.exists(p):
Expand Down
Loading