From 0697ff26c57f5bb374750a49409ef217b74a9569 Mon Sep 17 00:00:00 2001 From: troychiu Date: Mon, 13 Nov 2023 00:17:04 -0600 Subject: [PATCH 01/10] attribute access documentation Signed-off-by: troychiu --- examples/data_types_and_io/README.md | 1 + .../data_types_and_io/attribute_access.py | 185 ++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 examples/data_types_and_io/data_types_and_io/attribute_access.py diff --git a/examples/data_types_and_io/README.md b/examples/data_types_and_io/README.md index 9e1c29623..39650e239 100644 --- a/examples/data_types_and_io/README.md +++ b/examples/data_types_and_io/README.md @@ -141,6 +141,7 @@ file folder structured_dataset dataclass +attribute_access pytorch_type enum_type pickle_type diff --git a/examples/data_types_and_io/data_types_and_io/attribute_access.py b/examples/data_types_and_io/data_types_and_io/attribute_access.py new file mode 100644 index 000000000..214c29247 --- /dev/null +++ b/examples/data_types_and_io/data_types_and_io/attribute_access.py @@ -0,0 +1,185 @@ +# %% [markdown] +# (attribute_access)= +# +# # Attribute Access +# +# ```{eval-rst} +# .. tags:: Basic +# ``` +# +# Flyte allows users to access attributes directly on output promises for List, Dict, Dataclass, and combinations of them. This allows users to pass attributes of the output directly in workflows, making it more convenient to work with complex data structures. +# +# To begin, import the necessary dependencies. +# %% +from dataclasses import dataclass +from typing import Dict, List + +from dataclasses_json import dataclass_json +from flytekit import WorkflowFailurePolicy, task, workflow + + +# %% [markdown] +# ## List +# We can access the output list by index. +# :::{important} +# Currently, Flyte doesn't support accessing output promises by list slicing +# ::: +# %% +@task +def list_task() -> List[str]: + return ["a", "b"] + + +@task +def print_str(a: str): + print(a) + return + + +@workflow +def list_wf(): + o = list_task() + print_str(a=o[0]) + + +# %% [markdown] +# You can run the workflow locally. +# %% +if __name__ == "__main__": + list_wf() + +# %% [markdown] +# ## Dict +# We can access the output dict by key. +# %% +@task +def dict_task() -> Dict[str, str]: + return {"a": "b"} + + +@task +def print_str(a: str): + print(a) + return + + +@workflow +def dict_wf(): + o = dict_task() + print_str(a=o["a"]) + + +# %% [markdown] +# You can run the workflow locally. +# %% +if __name__ == "__main__": + dict_wf() + +# %% [markdown] +# ## Dataclass +# We can also access an attribute of a dataclass. +# %% +@dataclass_json +@dataclass +class foo: + a: str + + +@task +def dataclass_task() -> foo: + return foo(a="b") + + +@task +def print_str(a: str): + print(a) + return + + +@workflow +def dataclass_wf(): + o = dataclass_task() + print_str(a=o.a) + + +# %% [markdown] +# You can run the workflow locally. +# %% +if __name__ == "__main__": + dataclass_wf() + +# %% [markdown] +# ## Complex Examples +# The combinations of List, Dict, and Dataclass also work. +# %% +@task +def advance_task() -> (Dict[str, List[str]], List[Dict[str, str]], Dict[str, foo]): + return {"a": ["b"]}, [{"a": "b"}], {"a": foo(a="b")} + + +@task +def print_str(a: str): + print(a) + return + + +@task +def print_list(a: List[str]): + print(a) + + +@task +def print_dict(a: Dict[str, str]): + print(a) + + +@workflow +def advanced_workflow(): + dl, ld, dd = advance_task() + print_str(a=dl["a"][0]) + print_str(a=ld[0]["a"]) + print_str(a=dd["a"].a) + + print_list(a=dl["a"]) + print_dict(a=ld[0]) + + +# %% [markdown] +# You can run the workflow locally. +# %% +if __name__ == "__main__": + advanced_workflow() + + +# %% [markdown] +# ## Failed Examples +# The workflows will fail when there is an exception (e.g. out of range). +# %% +@task +def failed_task() -> (List[str], Dict[str, str], foo): + return ["a", "b"], {"a": "b"}, foo(a="b") + + +@task +def print_str(a: str): + print(a) + return + + +@workflow( + # The workflow will not fail if one of the nodes encounters an error, as long as there are other nodes that can still be executed. + failure_policy=WorkflowFailurePolicy.FAIL_AFTER_EXECUTABLE_NODES_COMPLETE +) +def failed_workflow(): + # This workflow is supposed to fail due to exceptions + l, d, f = failed_task() + print_str(a=l[100]) + print_str(a=d["b"]) + print_str(a=f.b) + + +# %% [markdown] +# You can run the workflow locally. +# %% +if __name__ == "__main__": + failed_workflow() From 9ced9e65efee312fc921afd7a2dcfea192a9847a Mon Sep 17 00:00:00 2001 From: troychiu Date: Mon, 13 Nov 2023 01:38:48 -0600 Subject: [PATCH 02/10] define print str as a common task Signed-off-by: troychiu --- .../data_types_and_io/attribute_access.py | 38 ++++--------------- 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/examples/data_types_and_io/data_types_and_io/attribute_access.py b/examples/data_types_and_io/data_types_and_io/attribute_access.py index 214c29247..5c1da7a2d 100644 --- a/examples/data_types_and_io/data_types_and_io/attribute_access.py +++ b/examples/data_types_and_io/data_types_and_io/attribute_access.py @@ -9,7 +9,7 @@ # # Flyte allows users to access attributes directly on output promises for List, Dict, Dataclass, and combinations of them. This allows users to pass attributes of the output directly in workflows, making it more convenient to work with complex data structures. # -# To begin, import the necessary dependencies. +# To begin, import the necessary dependencies and define a common task for later use. # %% from dataclasses import dataclass from typing import Dict, List @@ -18,6 +18,12 @@ from flytekit import WorkflowFailurePolicy, task, workflow +@task +def print_str(a: str): + print(a) + return + + # %% [markdown] # ## List # We can access the output list by index. @@ -30,12 +36,6 @@ def list_task() -> List[str]: return ["a", "b"] -@task -def print_str(a: str): - print(a) - return - - @workflow def list_wf(): o = list_task() @@ -57,12 +57,6 @@ def dict_task() -> Dict[str, str]: return {"a": "b"} -@task -def print_str(a: str): - print(a) - return - - @workflow def dict_wf(): o = dict_task() @@ -90,12 +84,6 @@ def dataclass_task() -> foo: return foo(a="b") -@task -def print_str(a: str): - print(a) - return - - @workflow def dataclass_wf(): o = dataclass_task() @@ -117,12 +105,6 @@ def advance_task() -> (Dict[str, List[str]], List[Dict[str, str]], Dict[str, foo return {"a": ["b"]}, [{"a": "b"}], {"a": foo(a="b")} -@task -def print_str(a: str): - print(a) - return - - @task def print_list(a: List[str]): print(a) @@ -160,12 +142,6 @@ def failed_task() -> (List[str], Dict[str, str], foo): return ["a", "b"], {"a": "b"}, foo(a="b") -@task -def print_str(a: str): - print(a) - return - - @workflow( # The workflow will not fail if one of the nodes encounters an error, as long as there are other nodes that can still be executed. failure_policy=WorkflowFailurePolicy.FAIL_AFTER_EXECUTABLE_NODES_COMPLETE From 20aa4fe4369ac7841ae8135b150f31ac09a39d81 Mon Sep 17 00:00:00 2001 From: troychiu Date: Mon, 13 Nov 2023 01:55:49 -0600 Subject: [PATCH 03/10] change dataclass to python dataclass to avoid error Signed-off-by: troychiu --- .../data_types_and_io/data_types_and_io/attribute_access.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/data_types_and_io/data_types_and_io/attribute_access.py b/examples/data_types_and_io/data_types_and_io/attribute_access.py index 5c1da7a2d..70d85c16a 100644 --- a/examples/data_types_and_io/data_types_and_io/attribute_access.py +++ b/examples/data_types_and_io/data_types_and_io/attribute_access.py @@ -70,7 +70,7 @@ def dict_wf(): dict_wf() # %% [markdown] -# ## Dataclass +# ## Python Dataclass # We can also access an attribute of a dataclass. # %% @dataclass_json From 32c7c681d45ccb6843f8c1707a7e83b7561c745a Mon Sep 17 00:00:00 2001 From: troychiu Date: Wed, 15 Nov 2023 18:44:34 -0600 Subject: [PATCH 04/10] remove execute fail workflow to pass ci Signed-off-by: troychiu --- .../data_types_and_io/data_types_and_io/attribute_access.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/examples/data_types_and_io/data_types_and_io/attribute_access.py b/examples/data_types_and_io/data_types_and_io/attribute_access.py index 70d85c16a..493ac0a3f 100644 --- a/examples/data_types_and_io/data_types_and_io/attribute_access.py +++ b/examples/data_types_and_io/data_types_and_io/attribute_access.py @@ -155,7 +155,4 @@ def failed_workflow(): # %% [markdown] -# You can run the workflow locally. -# %% -if __name__ == "__main__": - failed_workflow() +# failed_workflow should fail. From f2af3fbcf12718a925a6109bf9d2934eea9b1640 Mon Sep 17 00:00:00 2001 From: troychiu Date: Sun, 26 Nov 2023 17:36:14 -0800 Subject: [PATCH 05/10] update requirement Signed-off-by: troychiu --- dev-requirements.txt | 73 +++++++++++++-------------- docs-requirements.txt | 114 +++++++++++++++++++++--------------------- 2 files changed, 93 insertions(+), 94 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 174553f41..60a8e9d5f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,9 +6,9 @@ # adlfs==2023.10.0 # via flytekit -aiobotocore==2.7.0 +aiobotocore==2.5.4 # via s3fs -aiohttp==3.8.6 +aiohttp==3.9.1 # via # adlfs # aiobotocore @@ -20,8 +20,6 @@ aiosignal==1.3.1 # via aiohttp arrow==1.3.0 # via cookiecutter -async-timeout==4.0.3 - # via aiohttp attrs==23.1.0 # via aiohttp azure-core==1.29.5 @@ -33,19 +31,19 @@ azure-datalake-store==0.0.53 # via adlfs azure-identity==1.15.0 # via adlfs -azure-storage-blob==12.18.3 +azure-storage-blob==12.19.0 # via adlfs binaryornot==0.4.4 # via cookiecutter -black==23.10.1 +black==23.11.0 # via # -r dev-requirements.in # flake8-black -botocore==1.31.64 +botocore==1.31.17 # via aiobotocore cachetools==5.3.2 # via google-auth -certifi==2023.7.22 +certifi==2023.11.17 # via # kubernetes # requests @@ -58,9 +56,7 @@ cfgv==3.4.0 chardet==5.2.0 # via binaryornot charset-normalizer==3.3.2 - # via - # aiohttp - # requests + # via requests click==8.1.7 # via # black @@ -71,7 +67,7 @@ cloudpickle==3.0.0 # via flytekit codespell==2.2.6 # via -r dev-requirements.in -cookiecutter==2.4.0 +cookiecutter==2.5.0 # via flytekit coverage==7.3.2 # via -r dev-requirements.in @@ -109,29 +105,29 @@ flake8==6.1.0 # flake8-isort flake8-black==0.3.6 # via -r dev-requirements.in -flake8-isort==6.1.0 +flake8-isort==6.1.1 # via -r dev-requirements.in flyteidl==1.10.0 # via flytekit -flytekit==1.10.0 +flytekit==1.10.1 # via -r dev-requirements.in frozenlist==1.4.0 # via # aiohttp # aiosignal -fsspec==2023.10.0 +fsspec==2023.9.2 # via # adlfs # flytekit # gcsfs # s3fs -gcsfs==2023.10.0 +gcsfs==2023.9.2 # via flytekit gitdb==4.0.11 # via gitpython gitpython==3.1.40 # via flytekit -google-api-core==2.12.0 +google-api-core==2.14.0 # via # google-cloud-core # google-cloud-storage @@ -161,15 +157,15 @@ googleapis-common-protos==1.61.0 # flytekit # google-api-core # grpcio-status -grpcio==1.59.2 +grpcio==1.59.3 # via # flytekit # grpcio-status -grpcio-status==1.59.2 +grpcio-status==1.59.3 # via flytekit -identify==2.5.31 +identify==2.5.32 # via pre-commit -idna==3.4 +idna==3.6 # via # requests # yarl @@ -195,7 +191,7 @@ joblib==1.3.2 # via flytekit jsonpickle==3.0.2 # via flytekit -keyring==24.2.0 +keyring==24.3.0 # via flytekit kubernetes==28.1.0 # via flytekit @@ -214,7 +210,7 @@ marshmallow-enum==1.5.1 # flytekit marshmallow-jsonschema==0.13.0 # via flytekit -mashumaro==3.10 +mashumaro==3.11 # via # -r dev-requirements.in # flytekit @@ -226,7 +222,7 @@ mock==5.1.0 # via -r dev-requirements.in more-itertools==10.1.0 # via jaraco-classes -msal==1.24.1 +msal==1.25.0 # via # azure-datalake-store # azure-identity @@ -237,7 +233,7 @@ multidict==6.0.4 # via # aiohttp # yarl -mypy==1.6.1 +mypy==1.7.1 # via -r dev-requirements.in mypy-extensions==1.0.0 # via @@ -248,7 +244,7 @@ natsort==8.4.0 # via flytekit nodeenv==1.8.0 # via pre-commit -numpy==1.26.1 +numpy==1.26.2 # via # flytekit # pandas @@ -267,7 +263,7 @@ pandas==1.5.3 # via flytekit pathspec==0.11.2 # via black -platformdirs==3.11.0 +platformdirs==4.0.0 # via # black # virtualenv @@ -277,9 +273,10 @@ portalocker==2.8.2 # via msal-extensions pre-commit==3.5.0 # via -r dev-requirements.in -protobuf==4.25.0 +protobuf==4.24.4 # via # flyteidl + # flytekit # google-api-core # googleapis-common-protos # grpcio-status @@ -288,7 +285,7 @@ protoc-gen-swagger==0.1.0 # via flyteidl pyarrow==10.0.1 # via flytekit -pyasn1==0.5.0 +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -300,10 +297,12 @@ pycparser==2.21 # via cffi pyflakes==3.1.0 # via flake8 -pygments==2.16.1 +pygments==2.17.2 # via rich pyjwt[crypto]==2.8.0 - # via msal + # via + # msal + # pyjwt pyopenssl==23.3.0 # via flytekit pytest==7.4.3 @@ -352,7 +351,7 @@ requests-oauthlib==1.3.1 # via # google-auth-oauthlib # kubernetes -rich==13.6.0 +rich==13.7.0 # via # cookiecutter # flytekit @@ -361,7 +360,7 @@ rich-click==1.7.1 # via flytekit rsa==4.9 # via google-auth -s3fs==2023.10.0 +s3fs==2023.9.2 # via flytekit six==1.16.0 # via @@ -397,20 +396,20 @@ urllib3==1.26.18 # flytekit # kubernetes # requests -virtualenv==20.24.6 +virtualenv==20.24.7 # via pre-commit websocket-client==1.6.4 # via # docker # kubernetes -wheel==0.41.3 +wheel==0.42.0 # via flytekit -wrapt==1.15.0 +wrapt==1.16.0 # via # aiobotocore # deprecated # flytekit -yarl==1.9.2 +yarl==1.9.3 # via aiohttp zipp==3.17.0 # via importlib-metadata diff --git a/docs-requirements.txt b/docs-requirements.txt index b70cc0a77..76b919250 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -8,7 +8,7 @@ adlfs==2023.10.0 # via flytekit aiobotocore==2.5.4 # via s3fs -aiohttp==3.8.6 +aiohttp==3.9.1 # via # adlfs # aiobotocore @@ -23,9 +23,7 @@ alabaster==0.7.13 annotated-types==0.6.0 # via pydantic appnope==0.1.3 - # via - # ipykernel - # ipython + # via ipykernel arrow==1.3.0 # via cookiecutter astroid==3.0.1 @@ -34,8 +32,6 @@ astroid==3.0.1 # sphinx-autoapi asttokens==2.4.1 # via stack-data -async-timeout==4.0.3 - # via aiohttp attrs==23.1.0 # via # aiohttp @@ -52,7 +48,7 @@ azure-datalake-store==0.0.53 # via adlfs azure-identity==1.15.0 # via adlfs -azure-storage-blob==12.18.3 +azure-storage-blob==12.19.0 # via adlfs babel==2.13.1 # via sphinx @@ -66,7 +62,7 @@ botocore==1.31.17 # via aiobotocore cachetools==5.3.2 # via google-auth -certifi==2023.7.22 +certifi==2023.11.17 # via # kubernetes # requests @@ -79,9 +75,7 @@ cfgv==3.4.0 chardet==5.2.0 # via binaryornot charset-normalizer==3.3.2 - # via - # aiohttp - # requests + # via requests click==8.1.7 # via # cookiecutter @@ -92,13 +86,13 @@ cloudpickle==3.0.0 # via # flytekit # flytekitplugins-kfpytorch -comm==0.1.4 +comm==0.2.0 # via # ipykernel # ipywidgets -contourpy==1.1.1 +contourpy==1.2.0 # via matplotlib -cookiecutter==2.4.0 +cookiecutter==2.5.0 # via flytekit croniter==2.0.1 # via flytekit @@ -142,7 +136,7 @@ docutils==0.17.1 # sphinx-tabs executing==2.0.1 # via stack-data -fastjsonschema==2.18.1 +fastjsonschema==2.19.0 # via nbformat filelock==3.13.1 # via @@ -152,19 +146,19 @@ flyteidl==1.10.0 # via # flytekit # flytekitplugins-kfpytorch -flytekit==1.10.0 +flytekit==1.10.1 # via # -r docs-requirements.in # flytekitplugins-deck-standard # flytekitplugins-kfpytorch # flytekitplugins-sqlalchemy -flytekitplugins-deck-standard==1.10.0 +flytekitplugins-deck-standard==1.10.1 # via -r docs-requirements.in -flytekitplugins-kfpytorch==1.10.0 +flytekitplugins-kfpytorch==1.10.1 # via -r docs-requirements.in -flytekitplugins-sqlalchemy==1.10.0 +flytekitplugins-sqlalchemy==1.10.1 # via -r docs-requirements.in -fonttools==4.43.1 +fonttools==4.45.1 # via matplotlib frozenlist==1.4.0 # via @@ -186,7 +180,7 @@ gitdb==4.0.11 # via gitpython gitpython==3.1.40 # via flytekit -google-api-core==2.12.0 +google-api-core==2.14.0 # via # google-cloud-core # google-cloud-storage @@ -216,20 +210,20 @@ googleapis-common-protos==1.61.0 # flytekit # google-api-core # grpcio-status -grpcio==1.59.2 +grpcio==1.59.3 # via # -r docs-requirements.in # flytekit # grpcio-status -grpcio-status==1.59.2 +grpcio-status==1.59.3 # via # -r docs-requirements.in # flytekit htmlmin==0.1.12 # via ydata-profiling -identify==2.5.31 +identify==2.5.32 # via pre-commit -idna==3.4 +idna==3.6 # via # requests # yarl @@ -247,7 +241,7 @@ importlib-metadata==6.8.0 # myst-nb ipykernel==6.26.0 # via myst-nb -ipython==8.17.2 +ipython==8.18.0 # via # -r docs-requirements.in # ipykernel @@ -279,13 +273,13 @@ joblib==1.3.2 # scikit-learn jsonpickle==3.0.2 # via flytekit -jsonschema==4.19.2 +jsonschema==4.20.0 # via nbformat -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.1 # via jsonschema jupyter-cache==0.6.1 # via myst-nb -jupyter-client==8.5.0 +jupyter-client==8.6.0 # via # ipykernel # nbclient @@ -299,7 +293,7 @@ jupyterlab-widgets==3.0.9 # via ipywidgets jupytext==1.15.2 # via -r docs-requirements.in -keyring==24.2.0 +keyring==24.3.0 # via flytekit kiwisolver==1.4.5 # via matplotlib @@ -328,7 +322,7 @@ marshmallow-enum==1.5.1 # flytekit marshmallow-jsonschema==0.13.0 # via flytekit -mashumaro==3.10 +mashumaro==3.11 # via # -r docs-requirements.in # flytekit @@ -352,7 +346,7 @@ more-itertools==10.1.0 # via jaraco-classes mpmath==1.3.0 # via sympy -msal==1.24.1 +msal==1.25.0 # via # azure-datalake-store # azure-identity @@ -439,7 +433,7 @@ parso==0.8.3 # via jedi patsy==0.5.3 # via statsmodels -pexpect==4.8.0 +pexpect==4.9.0 # via ipython phik==0.12.3 # via ydata-profiling @@ -448,9 +442,10 @@ pillow==10.1.0 # -r docs-requirements.in # imagehash # matplotlib + # sphinx-gallery # visions # wordcloud -platformdirs==3.11.0 +platformdirs==4.0.0 # via # jupyter-core # virtualenv @@ -460,11 +455,12 @@ portalocker==2.8.2 # via msal-extensions pre-commit==3.5.0 # via sphinx-tags -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.41 # via ipython -protobuf==4.25.0 +protobuf==4.24.4 # via # flyteidl + # flytekit # google-api-core # googleapis-common-protos # grpcio-status @@ -481,7 +477,7 @@ pure-eval==0.2.2 # via stack-data pyarrow==10.0.1 # via flytekit -pyasn1==0.5.0 +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -489,11 +485,11 @@ pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pydantic==2.4.2 +pydantic==2.5.2 # via ydata-profiling -pydantic-core==2.10.1 +pydantic-core==2.14.5 # via pydantic -pygments==2.16.1 +pygments==2.17.2 # via # furo # ipython @@ -502,7 +498,9 @@ pygments==2.16.1 # sphinx-prompt # sphinx-tabs pyjwt[crypto]==2.8.0 - # via msal + # via + # msal + # pyjwt pyopenssl==23.3.0 # via flytekit pyparsing==3.1.1 @@ -528,7 +526,7 @@ pytz==2023.3.post1 # croniter # flytekit # pandas -pywavelets==1.4.1 +pywavelets==1.5.0 # via imagehash pyyaml==6.0.1 # via @@ -546,7 +544,7 @@ pyzmq==25.1.1 # via # ipykernel # jupyter-client -referencing==0.30.2 +referencing==0.31.0 # via # jsonschema # jsonschema-specifications @@ -572,14 +570,14 @@ requests-oauthlib==1.3.1 # via # google-auth-oauthlib # kubernetes -rich==13.6.0 +rich==13.7.0 # via # cookiecutter # flytekit # rich-click rich-click==1.7.1 # via flytekit -rpds-py==0.10.6 +rpds-py==0.13.1 # via # jsonschema # referencing @@ -589,7 +587,7 @@ s3fs==2023.9.2 # via flytekit scikit-learn==1.3.2 # via -r docs-requirements.in -scipy==1.11.3 +scipy==1.11.4 # via # imagehash # phik @@ -646,7 +644,7 @@ sphinx-copybutton==0.5.2 # via -r docs-requirements.in sphinx-fontawesome==0.0.6 # via -r docs-requirements.in -sphinx-gallery==0.14.0 +sphinx-gallery==0.15.0 # via -r docs-requirements.in sphinx-panels==0.6.0 # via -r docs-requirements.in @@ -678,7 +676,7 @@ sphinxcontrib-youtube==1.3.0 # via -r docs-requirements.in sphinxext-remoteliteralinclude==0.4.0 # via -r docs-requirements.in -sqlalchemy==2.0.22 +sqlalchemy==2.0.23 # via # flytekitplugins-sqlalchemy # jupyter-cache @@ -702,7 +700,7 @@ threadpoolctl==3.2.0 # via scikit-learn toml==0.10.2 # via jupytext -torch==2.1.0 +torch==2.1.1 # via -r docs-requirements.in tornado==6.3.3 # via @@ -751,30 +749,32 @@ urllib3==1.26.18 # flytekit # kubernetes # requests -virtualenv==20.24.6 +virtualenv==20.24.7 # via pre-commit -visions[type_image_path]==0.7.5 - # via ydata-profiling -wcwidth==0.2.9 +visions[type-image-path]==0.7.5 + # via + # visions + # ydata-profiling +wcwidth==0.2.12 # via prompt-toolkit websocket-client==1.6.4 # via # docker # kubernetes -wheel==0.41.3 +wheel==0.42.0 # via flytekit widgetsnbextension==4.0.9 # via ipywidgets wordcloud==1.9.2 # via ydata-profiling -wrapt==1.15.0 +wrapt==1.16.0 # via # aiobotocore # deprecated # flytekit -yarl==1.9.2 +yarl==1.9.3 # via aiohttp -ydata-profiling==4.6.1 +ydata-profiling==4.6.2 # via flytekitplugins-deck-standard zipp==3.17.0 # via importlib-metadata From a3be7b41abbfda61da3b37d3784e87fa2a207e50 Mon Sep 17 00:00:00 2001 From: troychiu Date: Sun, 26 Nov 2023 17:47:51 -0800 Subject: [PATCH 06/10] lint Signed-off-by: troychiu --- docs/getting_started/run_schedule.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting_started/run_schedule.md b/docs/getting_started/run_schedule.md index d8d650a7a..3aaca6625 100644 --- a/docs/getting_started/run_schedule.md +++ b/docs/getting_started/run_schedule.md @@ -73,7 +73,7 @@ environment, you can import and execute it directly: Before execute it directly, you need to register the workflow first. ```{prompt} bash $ -pyflyte register wf.py +pyflyte register wf.py ``` ```{code-block} python @@ -347,4 +347,4 @@ In this guide, you learned about how to: - Run tasks, workflows, and launch plans using `FlyteRemote`. - Create a cron schedule to run a launch plan at a specified time interval. -In the next guide, you'll learn how to visualize tasks using Flyte Decks. \ No newline at end of file +In the next guide, you'll learn how to visualize tasks using Flyte Decks. From 98544b3d3407929644581c3a8cda100826e3d3e7 Mon Sep 17 00:00:00 2001 From: troychiu Date: Mon, 27 Nov 2023 11:46:20 -0800 Subject: [PATCH 07/10] update flytekit version Signed-off-by: troychiu --- examples/data_types_and_io/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/data_types_and_io/Dockerfile b/examples/data_types_and_io/Dockerfile index 37f35a19f..6bee6a854 100644 --- a/examples/data_types_and_io/Dockerfile +++ b/examples/data_types_and_io/Dockerfile @@ -17,7 +17,7 @@ ENV VENV /opt/venv RUN python3 -m venv ${VENV} ENV PATH="${VENV}/bin:$PATH" -RUN pip install flytekit==1.10.0 torch +RUN pip install flytekit==1.10.1 torch # Copy the actual code COPY . /root From 6be7bf714b529d2dd95dda7fe2baa22081ca627d Mon Sep 17 00:00:00 2001 From: troychiu Date: Mon, 27 Nov 2023 18:17:25 -0800 Subject: [PATCH 08/10] Trigger Build Signed-off-by: troychiu From 3c0b247d42d69e3e6215dfcf1982b41054e0f3e4 Mon Sep 17 00:00:00 2001 From: troychiu Date: Wed, 29 Nov 2023 21:30:11 -0800 Subject: [PATCH 09/10] resolve suggestions Signed-off-by: troychiu --- .../data_types_and_io/attribute_access.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/data_types_and_io/data_types_and_io/attribute_access.py b/examples/data_types_and_io/data_types_and_io/attribute_access.py index 493ac0a3f..b72622abe 100644 --- a/examples/data_types_and_io/data_types_and_io/attribute_access.py +++ b/examples/data_types_and_io/data_types_and_io/attribute_access.py @@ -7,9 +7,9 @@ # .. tags:: Basic # ``` # -# Flyte allows users to access attributes directly on output promises for List, Dict, Dataclass, and combinations of them. This allows users to pass attributes of the output directly in workflows, making it more convenient to work with complex data structures. +# Flyte allows users to access attributes directly on output promises for List, Dict, Dataclass, and combinations of these types. This allows users to pass attributes of the output directly in workflows, making it more convenient to work with complex data structures. # -# To begin, import the necessary dependencies and define a common task for later use. +# First, import the necessary dependencies and define a common task for later use. # %% from dataclasses import dataclass from typing import Dict, List @@ -26,9 +26,9 @@ def print_str(a: str): # %% [markdown] # ## List -# We can access the output list by index. +# You can access the output list by index. # :::{important} -# Currently, Flyte doesn't support accessing output promises by list slicing +# Currently, Flyte doesn't support accessing output promises by list slicing. # ::: # %% @task @@ -50,7 +50,7 @@ def list_wf(): # %% [markdown] # ## Dict -# We can access the output dict by key. +# You can access the output dict by key. # %% @task def dict_task() -> Dict[str, str]: @@ -71,7 +71,7 @@ def dict_wf(): # %% [markdown] # ## Python Dataclass -# We can also access an attribute of a dataclass. +# You can also access an attribute of a dataclass. # %% @dataclass_json @dataclass @@ -98,7 +98,7 @@ def dataclass_wf(): # %% [markdown] # ## Complex Examples -# The combinations of List, Dict, and Dataclass also work. +# Combinations of List, Dict, and Dataclass also work. # %% @task def advance_task() -> (Dict[str, List[str]], List[Dict[str, str]], Dict[str, foo]): From 21e6927dce1c40813179a1a15444d46b1bc8749f Mon Sep 17 00:00:00 2001 From: troychiu Date: Thu, 30 Nov 2023 00:38:52 -0800 Subject: [PATCH 10/10] fix CI fail Signed-off-by: troychiu --- .../data_types_and_io/data_types_and_io/attribute_access.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/data_types_and_io/data_types_and_io/attribute_access.py b/examples/data_types_and_io/data_types_and_io/attribute_access.py index b72622abe..4a2621c86 100644 --- a/examples/data_types_and_io/data_types_and_io/attribute_access.py +++ b/examples/data_types_and_io/data_types_and_io/attribute_access.py @@ -151,7 +151,8 @@ def failed_workflow(): l, d, f = failed_task() print_str(a=l[100]) print_str(a=d["b"]) - print_str(a=f.b) + # This task will fail at compile time + # print_str(a=f.b) # %% [markdown]