diff --git a/dev-requirements.txt b/dev-requirements.txt index 174553f41..60a8e9d5f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,9 +6,9 @@ # adlfs==2023.10.0 # via flytekit -aiobotocore==2.7.0 +aiobotocore==2.5.4 # via s3fs -aiohttp==3.8.6 +aiohttp==3.9.1 # via # adlfs # aiobotocore @@ -20,8 +20,6 @@ aiosignal==1.3.1 # via aiohttp arrow==1.3.0 # via cookiecutter -async-timeout==4.0.3 - # via aiohttp attrs==23.1.0 # via aiohttp azure-core==1.29.5 @@ -33,19 +31,19 @@ azure-datalake-store==0.0.53 # via adlfs azure-identity==1.15.0 # via adlfs -azure-storage-blob==12.18.3 +azure-storage-blob==12.19.0 # via adlfs binaryornot==0.4.4 # via cookiecutter -black==23.10.1 +black==23.11.0 # via # -r dev-requirements.in # flake8-black -botocore==1.31.64 +botocore==1.31.17 # via aiobotocore cachetools==5.3.2 # via google-auth -certifi==2023.7.22 +certifi==2023.11.17 # via # kubernetes # requests @@ -58,9 +56,7 @@ cfgv==3.4.0 chardet==5.2.0 # via binaryornot charset-normalizer==3.3.2 - # via - # aiohttp - # requests + # via requests click==8.1.7 # via # black @@ -71,7 +67,7 @@ cloudpickle==3.0.0 # via flytekit codespell==2.2.6 # via -r dev-requirements.in -cookiecutter==2.4.0 +cookiecutter==2.5.0 # via flytekit coverage==7.3.2 # via -r dev-requirements.in @@ -109,29 +105,29 @@ flake8==6.1.0 # flake8-isort flake8-black==0.3.6 # via -r dev-requirements.in -flake8-isort==6.1.0 +flake8-isort==6.1.1 # via -r dev-requirements.in flyteidl==1.10.0 # via flytekit -flytekit==1.10.0 +flytekit==1.10.1 # via -r dev-requirements.in frozenlist==1.4.0 # via # aiohttp # aiosignal -fsspec==2023.10.0 +fsspec==2023.9.2 # via # adlfs # flytekit # gcsfs # s3fs -gcsfs==2023.10.0 +gcsfs==2023.9.2 # via flytekit gitdb==4.0.11 # via gitpython gitpython==3.1.40 # via flytekit -google-api-core==2.12.0 +google-api-core==2.14.0 # via # google-cloud-core # google-cloud-storage @@ -161,15 +157,15 @@ googleapis-common-protos==1.61.0 # flytekit # google-api-core # grpcio-status -grpcio==1.59.2 +grpcio==1.59.3 # via # flytekit # grpcio-status -grpcio-status==1.59.2 +grpcio-status==1.59.3 # via flytekit -identify==2.5.31 +identify==2.5.32 # via pre-commit -idna==3.4 +idna==3.6 # via # requests # yarl @@ -195,7 +191,7 @@ joblib==1.3.2 # via flytekit jsonpickle==3.0.2 # via flytekit -keyring==24.2.0 +keyring==24.3.0 # via flytekit kubernetes==28.1.0 # via flytekit @@ -214,7 +210,7 @@ marshmallow-enum==1.5.1 # flytekit marshmallow-jsonschema==0.13.0 # via flytekit -mashumaro==3.10 +mashumaro==3.11 # via # -r dev-requirements.in # flytekit @@ -226,7 +222,7 @@ mock==5.1.0 # via -r dev-requirements.in more-itertools==10.1.0 # via jaraco-classes -msal==1.24.1 +msal==1.25.0 # via # azure-datalake-store # azure-identity @@ -237,7 +233,7 @@ multidict==6.0.4 # via # aiohttp # yarl -mypy==1.6.1 +mypy==1.7.1 # via -r dev-requirements.in mypy-extensions==1.0.0 # via @@ -248,7 +244,7 @@ natsort==8.4.0 # via flytekit nodeenv==1.8.0 # via pre-commit -numpy==1.26.1 +numpy==1.26.2 # via # flytekit # pandas @@ -267,7 +263,7 @@ pandas==1.5.3 # via flytekit pathspec==0.11.2 # via black -platformdirs==3.11.0 +platformdirs==4.0.0 # via # black # virtualenv @@ -277,9 +273,10 @@ portalocker==2.8.2 # via msal-extensions pre-commit==3.5.0 # via -r dev-requirements.in -protobuf==4.25.0 +protobuf==4.24.4 # via # flyteidl + # flytekit # google-api-core # googleapis-common-protos # grpcio-status @@ -288,7 +285,7 @@ protoc-gen-swagger==0.1.0 # via flyteidl pyarrow==10.0.1 # via flytekit -pyasn1==0.5.0 +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -300,10 +297,12 @@ pycparser==2.21 # via cffi pyflakes==3.1.0 # via flake8 -pygments==2.16.1 +pygments==2.17.2 # via rich pyjwt[crypto]==2.8.0 - # via msal + # via + # msal + # pyjwt pyopenssl==23.3.0 # via flytekit pytest==7.4.3 @@ -352,7 +351,7 @@ requests-oauthlib==1.3.1 # via # google-auth-oauthlib # kubernetes -rich==13.6.0 +rich==13.7.0 # via # cookiecutter # flytekit @@ -361,7 +360,7 @@ rich-click==1.7.1 # via flytekit rsa==4.9 # via google-auth -s3fs==2023.10.0 +s3fs==2023.9.2 # via flytekit six==1.16.0 # via @@ -397,20 +396,20 @@ urllib3==1.26.18 # flytekit # kubernetes # requests -virtualenv==20.24.6 +virtualenv==20.24.7 # via pre-commit websocket-client==1.6.4 # via # docker # kubernetes -wheel==0.41.3 +wheel==0.42.0 # via flytekit -wrapt==1.15.0 +wrapt==1.16.0 # via # aiobotocore # deprecated # flytekit -yarl==1.9.2 +yarl==1.9.3 # via aiohttp zipp==3.17.0 # via importlib-metadata diff --git a/docs-requirements.txt b/docs-requirements.txt index b70cc0a77..76b919250 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -8,7 +8,7 @@ adlfs==2023.10.0 # via flytekit aiobotocore==2.5.4 # via s3fs -aiohttp==3.8.6 +aiohttp==3.9.1 # via # adlfs # aiobotocore @@ -23,9 +23,7 @@ alabaster==0.7.13 annotated-types==0.6.0 # via pydantic appnope==0.1.3 - # via - # ipykernel - # ipython + # via ipykernel arrow==1.3.0 # via cookiecutter astroid==3.0.1 @@ -34,8 +32,6 @@ astroid==3.0.1 # sphinx-autoapi asttokens==2.4.1 # via stack-data -async-timeout==4.0.3 - # via aiohttp attrs==23.1.0 # via # aiohttp @@ -52,7 +48,7 @@ azure-datalake-store==0.0.53 # via adlfs azure-identity==1.15.0 # via adlfs -azure-storage-blob==12.18.3 +azure-storage-blob==12.19.0 # via adlfs babel==2.13.1 # via sphinx @@ -66,7 +62,7 @@ botocore==1.31.17 # via aiobotocore cachetools==5.3.2 # via google-auth -certifi==2023.7.22 +certifi==2023.11.17 # via # kubernetes # requests @@ -79,9 +75,7 @@ cfgv==3.4.0 chardet==5.2.0 # via binaryornot charset-normalizer==3.3.2 - # via - # aiohttp - # requests + # via requests click==8.1.7 # via # cookiecutter @@ -92,13 +86,13 @@ cloudpickle==3.0.0 # via # flytekit # flytekitplugins-kfpytorch -comm==0.1.4 +comm==0.2.0 # via # ipykernel # ipywidgets -contourpy==1.1.1 +contourpy==1.2.0 # via matplotlib -cookiecutter==2.4.0 +cookiecutter==2.5.0 # via flytekit croniter==2.0.1 # via flytekit @@ -142,7 +136,7 @@ docutils==0.17.1 # sphinx-tabs executing==2.0.1 # via stack-data -fastjsonschema==2.18.1 +fastjsonschema==2.19.0 # via nbformat filelock==3.13.1 # via @@ -152,19 +146,19 @@ flyteidl==1.10.0 # via # flytekit # flytekitplugins-kfpytorch -flytekit==1.10.0 +flytekit==1.10.1 # via # -r docs-requirements.in # flytekitplugins-deck-standard # flytekitplugins-kfpytorch # flytekitplugins-sqlalchemy -flytekitplugins-deck-standard==1.10.0 +flytekitplugins-deck-standard==1.10.1 # via -r docs-requirements.in -flytekitplugins-kfpytorch==1.10.0 +flytekitplugins-kfpytorch==1.10.1 # via -r docs-requirements.in -flytekitplugins-sqlalchemy==1.10.0 +flytekitplugins-sqlalchemy==1.10.1 # via -r docs-requirements.in -fonttools==4.43.1 +fonttools==4.45.1 # via matplotlib frozenlist==1.4.0 # via @@ -186,7 +180,7 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.40 # via flytekit -google-api-core==2.12.0 +google-api-core==2.14.0 # via # google-cloud-core # google-cloud-storage @@ -216,20 +210,20 @@ googleapis-common-protos==1.61.0 # flytekit # google-api-core # grpcio-status -grpcio==1.59.2 +grpcio==1.59.3 # via # -r docs-requirements.in # flytekit # grpcio-status -grpcio-status==1.59.2 +grpcio-status==1.59.3 # via # -r docs-requirements.in # flytekit htmlmin==0.1.12 # via ydata-profiling -identify==2.5.31 +identify==2.5.32 # via pre-commit -idna==3.4 +idna==3.6 # via # requests # yarl @@ -247,7 +241,7 @@ importlib-metadata==6.8.0 # myst-nb ipykernel==6.26.0 # via myst-nb -ipython==8.17.2 +ipython==8.18.0 # via # -r docs-requirements.in # ipykernel @@ -279,13 +273,13 @@ joblib==1.3.2 # scikit-learn jsonpickle==3.0.2 # via flytekit -jsonschema==4.19.2 +jsonschema==4.20.0 # via nbformat -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.1 # via jsonschema jupyter-cache==0.6.1 # via myst-nb -jupyter-client==8.5.0 +jupyter-client==8.6.0 # via # ipykernel # nbclient @@ -299,7 +293,7 @@ jupyterlab-widgets==3.0.9 # via ipywidgets jupytext==1.15.2 # via -r docs-requirements.in -keyring==24.2.0 +keyring==24.3.0 # via flytekit kiwisolver==1.4.5 # via matplotlib @@ -328,7 +322,7 @@ marshmallow-enum==1.5.1 # flytekit marshmallow-jsonschema==0.13.0 # via flytekit -mashumaro==3.10 +mashumaro==3.11 # via # -r docs-requirements.in # flytekit @@ -352,7 +346,7 @@ more-itertools==10.1.0 # via jaraco-classes mpmath==1.3.0 # via sympy -msal==1.24.1 +msal==1.25.0 # via # azure-datalake-store # azure-identity @@ -439,7 +433,7 @@ parso==0.8.3 # via jedi patsy==0.5.3 # via statsmodels -pexpect==4.8.0 +pexpect==4.9.0 # via ipython phik==0.12.3 # via ydata-profiling @@ -448,9 +442,10 @@ pillow==10.1.0 # -r docs-requirements.in # imagehash # matplotlib + # sphinx-gallery # visions # wordcloud -platformdirs==3.11.0 +platformdirs==4.0.0 # via # jupyter-core # virtualenv @@ -460,11 +455,12 @@ portalocker==2.8.2 # via msal-extensions pre-commit==3.5.0 # via sphinx-tags -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.41 # via ipython -protobuf==4.25.0 +protobuf==4.24.4 # via # flyteidl + # flytekit # google-api-core # googleapis-common-protos # grpcio-status @@ -481,7 +477,7 @@ pure-eval==0.2.2 # via stack-data pyarrow==10.0.1 # via flytekit -pyasn1==0.5.0 +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -489,11 +485,11 @@ pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pydantic==2.4.2 +pydantic==2.5.2 # via ydata-profiling -pydantic-core==2.10.1 +pydantic-core==2.14.5 # via pydantic -pygments==2.16.1 +pygments==2.17.2 # via # furo # ipython @@ -502,7 +498,9 @@ pygments==2.16.1 # sphinx-prompt # sphinx-tabs pyjwt[crypto]==2.8.0 - # via msal + # via + # msal + # pyjwt pyopenssl==23.3.0 # via flytekit pyparsing==3.1.1 @@ -528,7 +526,7 @@ pytz==2023.3.post1 # croniter # flytekit # pandas -pywavelets==1.4.1 +pywavelets==1.5.0 # via imagehash pyyaml==6.0.1 # via @@ -546,7 +544,7 @@ pyzmq==25.1.1 # via # ipykernel # jupyter-client -referencing==0.30.2 +referencing==0.31.0 # via # jsonschema # jsonschema-specifications @@ -572,14 +570,14 @@ requests-oauthlib==1.3.1 # via # google-auth-oauthlib # kubernetes -rich==13.6.0 +rich==13.7.0 # via # cookiecutter # flytekit # rich-click rich-click==1.7.1 # via flytekit -rpds-py==0.10.6 +rpds-py==0.13.1 # via # jsonschema # referencing @@ -589,7 +587,7 @@ s3fs==2023.9.2 # via flytekit scikit-learn==1.3.2 # via -r docs-requirements.in -scipy==1.11.3 +scipy==1.11.4 # via # imagehash # phik @@ -646,7 +644,7 @@ sphinx-copybutton==0.5.2 # via -r docs-requirements.in sphinx-fontawesome==0.0.6 # via -r docs-requirements.in -sphinx-gallery==0.14.0 +sphinx-gallery==0.15.0 # via -r docs-requirements.in sphinx-panels==0.6.0 # via -r docs-requirements.in @@ -678,7 +676,7 @@ sphinxcontrib-youtube==1.3.0 # via -r docs-requirements.in sphinxext-remoteliteralinclude==0.4.0 # via -r docs-requirements.in -sqlalchemy==2.0.22 +sqlalchemy==2.0.23 # via # flytekitplugins-sqlalchemy # jupyter-cache @@ -702,7 +700,7 @@ threadpoolctl==3.2.0 # via scikit-learn toml==0.10.2 # via jupytext -torch==2.1.0 +torch==2.1.1 # via -r docs-requirements.in tornado==6.3.3 # via @@ -751,30 +749,32 @@ urllib3==1.26.18 # flytekit # kubernetes # requests -virtualenv==20.24.6 +virtualenv==20.24.7 # via pre-commit -visions[type_image_path]==0.7.5 - # via ydata-profiling -wcwidth==0.2.9 +visions[type-image-path]==0.7.5 + # via + # visions + # ydata-profiling +wcwidth==0.2.12 # via prompt-toolkit websocket-client==1.6.4 # via # docker # kubernetes -wheel==0.41.3 +wheel==0.42.0 # via flytekit widgetsnbextension==4.0.9 # via ipywidgets wordcloud==1.9.2 # via ydata-profiling -wrapt==1.15.0 +wrapt==1.16.0 # via # aiobotocore # deprecated # flytekit -yarl==1.9.2 +yarl==1.9.3 # via aiohttp -ydata-profiling==4.6.1 +ydata-profiling==4.6.2 # via flytekitplugins-deck-standard zipp==3.17.0 # via importlib-metadata diff --git a/docs/getting_started/run_schedule.md b/docs/getting_started/run_schedule.md index d8d650a7a..3aaca6625 100644 --- a/docs/getting_started/run_schedule.md +++ b/docs/getting_started/run_schedule.md @@ -73,7 +73,7 @@ environment, you can import and execute it directly: Before execute it directly, you need to register the workflow first. ```{prompt} bash $ -pyflyte register wf.py +pyflyte register wf.py ``` ```{code-block} python @@ -347,4 +347,4 @@ In this guide, you learned about how to: - Run tasks, workflows, and launch plans using `FlyteRemote`. - Create a cron schedule to run a launch plan at a specified time interval. -In the next guide, you'll learn how to visualize tasks using Flyte Decks. \ No newline at end of file +In the next guide, you'll learn how to visualize tasks using Flyte Decks. diff --git a/examples/data_types_and_io/Dockerfile b/examples/data_types_and_io/Dockerfile index 37f35a19f..6bee6a854 100644 --- a/examples/data_types_and_io/Dockerfile +++ b/examples/data_types_and_io/Dockerfile @@ -17,7 +17,7 @@ ENV VENV /opt/venv RUN python3 -m venv ${VENV} ENV PATH="${VENV}/bin:$PATH" -RUN pip install flytekit==1.10.0 torch +RUN pip install flytekit==1.10.1 torch # Copy the actual code COPY . /root diff --git a/examples/data_types_and_io/README.md b/examples/data_types_and_io/README.md index 9e1c29623..39650e239 100644 --- a/examples/data_types_and_io/README.md +++ b/examples/data_types_and_io/README.md @@ -141,6 +141,7 @@ file folder structured_dataset dataclass +attribute_access pytorch_type enum_type pickle_type diff --git a/examples/data_types_and_io/data_types_and_io/attribute_access.py b/examples/data_types_and_io/data_types_and_io/attribute_access.py new file mode 100644 index 000000000..4a2621c86 --- /dev/null +++ b/examples/data_types_and_io/data_types_and_io/attribute_access.py @@ -0,0 +1,159 @@ +# %% [markdown] +# (attribute_access)= +# +# # Attribute Access +# +# ```{eval-rst} +# .. tags:: Basic +# ``` +# +# Flyte allows users to access attributes directly on output promises for List, Dict, Dataclass, and combinations of these types. This allows users to pass attributes of the output directly in workflows, making it more convenient to work with complex data structures. +# +# First, import the necessary dependencies and define a common task for later use. +# %% +from dataclasses import dataclass +from typing import Dict, List + +from dataclasses_json import dataclass_json +from flytekit import WorkflowFailurePolicy, task, workflow + + +@task +def print_str(a: str): + print(a) + return + + +# %% [markdown] +# ## List +# You can access the output list by index. +# :::{important} +# Currently, Flyte doesn't support accessing output promises by list slicing. +# ::: +# %% +@task +def list_task() -> List[str]: + return ["a", "b"] + + +@workflow +def list_wf(): + o = list_task() + print_str(a=o[0]) + + +# %% [markdown] +# You can run the workflow locally. +# %% +if __name__ == "__main__": + list_wf() + +# %% [markdown] +# ## Dict +# You can access the output dict by key. +# %% +@task +def dict_task() -> Dict[str, str]: + return {"a": "b"} + + +@workflow +def dict_wf(): + o = dict_task() + print_str(a=o["a"]) + + +# %% [markdown] +# You can run the workflow locally. +# %% +if __name__ == "__main__": + dict_wf() + +# %% [markdown] +# ## Python Dataclass +# You can also access an attribute of a dataclass. +# %% +@dataclass_json +@dataclass +class foo: + a: str + + +@task +def dataclass_task() -> foo: + return foo(a="b") + + +@workflow +def dataclass_wf(): + o = dataclass_task() + print_str(a=o.a) + + +# %% [markdown] +# You can run the workflow locally. +# %% +if __name__ == "__main__": + dataclass_wf() + +# %% [markdown] +# ## Complex Examples +# Combinations of List, Dict, and Dataclass also work. +# %% +@task +def advance_task() -> (Dict[str, List[str]], List[Dict[str, str]], Dict[str, foo]): + return {"a": ["b"]}, [{"a": "b"}], {"a": foo(a="b")} + + +@task +def print_list(a: List[str]): + print(a) + + +@task +def print_dict(a: Dict[str, str]): + print(a) + + +@workflow +def advanced_workflow(): + dl, ld, dd = advance_task() + print_str(a=dl["a"][0]) + print_str(a=ld[0]["a"]) + print_str(a=dd["a"].a) + + print_list(a=dl["a"]) + print_dict(a=ld[0]) + + +# %% [markdown] +# You can run the workflow locally. +# %% +if __name__ == "__main__": + advanced_workflow() + + +# %% [markdown] +# ## Failed Examples +# The workflows will fail when there is an exception (e.g. out of range). +# %% +@task +def failed_task() -> (List[str], Dict[str, str], foo): + return ["a", "b"], {"a": "b"}, foo(a="b") + + +@workflow( + # The workflow will not fail if one of the nodes encounters an error, as long as there are other nodes that can still be executed. + failure_policy=WorkflowFailurePolicy.FAIL_AFTER_EXECUTABLE_NODES_COMPLETE +) +def failed_workflow(): + # This workflow is supposed to fail due to exceptions + l, d, f = failed_task() + print_str(a=l[100]) + print_str(a=d["b"]) + # This task will fail at compile time + # print_str(a=f.b) + + +# %% [markdown] +# failed_workflow should fail.