From 263b41085869b39005b81c0a456688b324986e80 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 23 May 2024 11:44:18 -0700 Subject: [PATCH 1/7] Prevent mutable default arguments Signed-off-by: Eduardo Apolinario --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e3d311a73a..ab3a3db6eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,7 @@ branch = true [tool.ruff] line-length = 120 -lint.select = ["E", "W", "F", "I"] +lint.select = ["E", "W", "F", "I", "B006"] lint.ignore = [ # Whitespace before '{symbol}' "E203", @@ -133,6 +133,9 @@ lint.ignore = [ # Do not assign a lambda expression, use a def "E731", ] +extend-exclude = [ + "tests/" +] [tool.ruff.lint.extend-per-file-ignores] "*/__init__.py" = [ From 349f2dbe8bc60da397e2e2131878752d8cf9ed6e Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 4 Jun 2024 14:37:37 -0700 Subject: [PATCH 2/7] Do not lint tests Signed-off-by: Eduardo Apolinario --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ab3a3db6eb..9d25206298 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -134,7 +134,8 @@ lint.ignore = [ "E731", ] extend-exclude = [ - "tests/" + "tests/", + "**/tests/**", ] [tool.ruff.lint.extend-per-file-ignores] From 62f1639438d548fc68f20b7276c3d07c59bdd414 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 4 Jun 2024 14:38:02 -0700 Subject: [PATCH 3/7] Bump precommit hook version Signed-off-by: Eduardo Apolinario --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 703bcda938..a0fc842ba2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.2.2 + rev: v0.4.7 hooks: # Run the linter. - id: ruff From 4d09fb3ee17873e9b163271deb9762b13ff6ceb2 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 4 Jun 2024 14:38:34 -0700 Subject: [PATCH 4/7] Remove default value for dictionary in Spark class Signed-off-by: Eduardo Apolinario --- plugins/flytekit-spark/flytekitplugins/spark/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/flytekit-spark/flytekitplugins/spark/models.py b/plugins/flytekit-spark/flytekitplugins/spark/models.py index 28e67ac631..75142f040e 100644 --- a/plugins/flytekit-spark/flytekitplugins/spark/models.py +++ b/plugins/flytekit-spark/flytekitplugins/spark/models.py @@ -25,7 +25,7 @@ def __init__( spark_conf: Dict[str, str], hadoop_conf: Dict[str, str], executor_path: str, - databricks_conf: Dict[str, Dict[str, Dict]] = {}, + databricks_conf: Dict[str, Dict[str, Dict]], databricks_token: Optional[str] = None, databricks_instance: Optional[str] = None, ): From de8798692669411af5880ac914def032bac6a4ab Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 4 Jun 2024 14:39:04 -0700 Subject: [PATCH 5/7] Run precommit hook locally Signed-off-by: Eduardo Apolinario --- flytekit/__init__.py | 1 + flytekit/clients/auth/authenticator.py | 6 ++--- flytekit/configuration/__init__.py | 1 + flytekit/configuration/plugin.py | 1 + flytekit/core/artifact.py | 9 +++---- flytekit/core/context_manager.py | 3 +-- flytekit/core/data_persistence.py | 1 + flytekit/core/dynamic_workflow_task.py | 1 + flytekit/core/interface.py | 3 +-- flytekit/core/legacy_map_task.py | 1 + flytekit/core/local_fsspec.py | 1 + flytekit/core/notification.py | 1 + flytekit/core/promise.py | 24 +++++++------------ flytekit/core/reference_entity.py | 3 +-- flytekit/core/schedule.py | 3 +-- flytekit/core/task.py | 6 ++--- flytekit/core/type_engine.py | 3 +-- flytekit/core/workflow.py | 6 ++--- flytekit/extras/accelerators.py | 4 ++-- flytekit/extras/pytorch/__init__.py | 1 + flytekit/extras/sklearn/__init__.py | 1 + flytekit/remote/entities.py | 1 + flytekit/remote/executions.py | 6 ++--- flytekit/remote/remote.py | 1 + flytekit/remote/remote_callable.py | 3 +-- flytekit/types/directory/types.py | 3 +-- flytekit/types/file/__init__.py | 1 + flytekit/types/file/file.py | 3 +-- flytekit/types/schema/types.py | 9 +++---- flytekit/types/structured/__init__.py | 1 - .../types/structured/structured_dataset.py | 11 ++++----- .../flytekitplugins/async_fsspec/__init__.py | 1 + .../flytekitplugins/kfmpi/task.py | 1 + .../flytekit-kf-mpi/tests/test_mpi_task.py | 3 +-- .../kfpytorch/error_handling.py | 1 + .../flytekitplugins/kfpytorch/task.py | 1 + .../flytekitplugins/kftensorflow/task.py | 1 + .../flytekitplugins/modin/schema.py | 6 ++--- .../flytekitplugins/pandera/schema.py | 6 ++--- .../pydantic/basemodel_transformer.py | 2 +- .../flytekitplugins/pydantic/serialization.py | 1 + 41 files changed, 64 insertions(+), 78 deletions(-) diff --git a/flytekit/__init__.py b/flytekit/__init__.py index 63ad935b47..33bdad747a 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -202,6 +202,7 @@ SourceCode """ + import os import sys from typing import Generator diff --git a/flytekit/clients/auth/authenticator.py b/flytekit/clients/auth/authenticator.py index 0ed780509e..95a89422be 100644 --- a/flytekit/clients/auth/authenticator.py +++ b/flytekit/clients/auth/authenticator.py @@ -35,8 +35,7 @@ class ClientConfigStore(object): """ @abstractmethod - def get_client_config(self) -> ClientConfig: - ... + def get_client_config(self) -> ClientConfig: ... class StaticClientConfigStore(ClientConfigStore): @@ -81,8 +80,7 @@ def fetch_grpc_call_auth_metadata(self) -> typing.Optional[typing.Tuple[str, str return None @abstractmethod - def refresh_credentials(self): - ... + def refresh_credentials(self): ... class PKCEAuthenticator(Authenticator): diff --git a/flytekit/configuration/__init__.py b/flytekit/configuration/__init__.py index c29bd71c88..97a9940425 100644 --- a/flytekit/configuration/__init__.py +++ b/flytekit/configuration/__init__.py @@ -126,6 +126,7 @@ ~DataConfig """ + from __future__ import annotations import base64 diff --git a/flytekit/configuration/plugin.py b/flytekit/configuration/plugin.py index cc8750deaa..3d43844d39 100644 --- a/flytekit/configuration/plugin.py +++ b/flytekit/configuration/plugin.py @@ -17,6 +17,7 @@ my_plugin = "my_module:MyCustomPlugin" ``` """ + from typing import Optional, Protocol, runtime_checkable from click import Group diff --git a/flytekit/core/artifact.py b/flytekit/core/artifact.py index f2e08042bc..954151504f 100644 --- a/flytekit/core/artifact.py +++ b/flytekit/core/artifact.py @@ -607,14 +607,11 @@ class ArtifactSerializationHandler(typing.Protocol): This protocol defines the interface for serializing artifact-related entities down to Flyte IDL. """ - def partitions_to_idl(self, p: Optional[Partitions], **kwargs) -> Optional[art_id.Partitions]: - ... + def partitions_to_idl(self, p: Optional[Partitions], **kwargs) -> Optional[art_id.Partitions]: ... - def time_partition_to_idl(self, tp: Optional[TimePartition], **kwargs) -> Optional[art_id.TimePartition]: - ... + def time_partition_to_idl(self, tp: Optional[TimePartition], **kwargs) -> Optional[art_id.TimePartition]: ... - def artifact_query_to_idl(self, aq: ArtifactQuery, **kwargs) -> art_id.ArtifactQuery: - ... + def artifact_query_to_idl(self, aq: ArtifactQuery, **kwargs) -> art_id.ArtifactQuery: ... class DefaultArtifactSerializationHandler(ArtifactSerializationHandler): diff --git a/flytekit/core/context_manager.py b/flytekit/core/context_manager.py index a31d058774..c51b60c1c9 100644 --- a/flytekit/core/context_manager.py +++ b/flytekit/core/context_manager.py @@ -560,8 +560,7 @@ class SerializableToString(typing.Protocol): and then added to a literal's metadata. """ - def serialize_to_string(self, ctx: FlyteContext, variable_name: str) -> typing.Tuple[str, str]: - ... + def serialize_to_string(self, ctx: FlyteContext, variable_name: str) -> typing.Tuple[str, str]: ... @dataclass diff --git a/flytekit/core/data_persistence.py b/flytekit/core/data_persistence.py index 492362b819..035698181e 100644 --- a/flytekit/core/data_persistence.py +++ b/flytekit/core/data_persistence.py @@ -17,6 +17,7 @@ FileAccessProvider """ + import io import os import pathlib diff --git a/flytekit/core/dynamic_workflow_task.py b/flytekit/core/dynamic_workflow_task.py index a0f84927bf..a9ff5055db 100644 --- a/flytekit/core/dynamic_workflow_task.py +++ b/flytekit/core/dynamic_workflow_task.py @@ -12,6 +12,7 @@ dynamic workflows to under fifty tasks. For large-scale identical runs, we recommend the upcoming map task. """ + import functools from flytekit.core import task diff --git a/flytekit/core/interface.py b/flytekit/core/interface.py index c139641278..13b6af2d4b 100644 --- a/flytekit/core/interface.py +++ b/flytekit/core/interface.py @@ -109,8 +109,7 @@ def runs_before(self, *args, **kwargs): where runs_before is manually called. """ - def __rshift__(self, *args, **kwargs): - ... # See runs_before + def __rshift__(self, *args, **kwargs): ... # See runs_before self._output_tuple_class = Output self._docstring = docstring diff --git a/flytekit/core/legacy_map_task.py b/flytekit/core/legacy_map_task.py index fe8d353027..99c67ad12c 100644 --- a/flytekit/core/legacy_map_task.py +++ b/flytekit/core/legacy_map_task.py @@ -2,6 +2,7 @@ Flytekit map tasks specify how to run a single task across a list of inputs. Map tasks themselves are constructed with a reference task as well as run-time parameters that limit execution concurrency and failure tolerations. """ + import functools import hashlib import logging diff --git a/flytekit/core/local_fsspec.py b/flytekit/core/local_fsspec.py index b452b3006e..91fe93ad6f 100644 --- a/flytekit/core/local_fsspec.py +++ b/flytekit/core/local_fsspec.py @@ -14,6 +14,7 @@ FlyteLocalFileSystem """ + import os from fsspec.implementations.local import LocalFileSystem diff --git a/flytekit/core/notification.py b/flytekit/core/notification.py index cecfe43367..c964c67568 100644 --- a/flytekit/core/notification.py +++ b/flytekit/core/notification.py @@ -15,6 +15,7 @@ .. autoclass:: flytekit.core.notification.Notification """ + from typing import List from flytekit.models import common as _common_model diff --git a/flytekit/core/promise.py b/flytekit/core/promise.py index 220fc3fb89..6190ae5af5 100644 --- a/flytekit/core/promise.py +++ b/flytekit/core/promise.py @@ -891,28 +891,22 @@ def with_attr(self, key) -> NodeOutput: class SupportsNodeCreation(Protocol): @property - def name(self) -> str: - ... + def name(self) -> str: ... @property - def python_interface(self) -> flyte_interface.Interface: - ... + def python_interface(self) -> flyte_interface.Interface: ... - def construct_node_metadata(self) -> _workflow_model.NodeMetadata: - ... + def construct_node_metadata(self) -> _workflow_model.NodeMetadata: ... class HasFlyteInterface(Protocol): @property - def name(self) -> str: - ... + def name(self) -> str: ... @property - def interface(self) -> _interface_models.TypedInterface: - ... + def interface(self) -> _interface_models.TypedInterface: ... - def construct_node_metadata(self) -> _workflow_model.NodeMetadata: - ... + def construct_node_metadata(self) -> _workflow_model.NodeMetadata: ... def extract_obj_name(name: str) -> str: @@ -1139,11 +1133,9 @@ def create_and_link_node( class LocallyExecutable(Protocol): - def local_execute(self, ctx: FlyteContext, **kwargs) -> Union[Tuple[Promise], Promise, VoidPromise, None]: - ... + def local_execute(self, ctx: FlyteContext, **kwargs) -> Union[Tuple[Promise], Promise, VoidPromise, None]: ... - def local_execution_mode(self) -> ExecutionState.Mode: - ... + def local_execution_mode(self) -> ExecutionState.Mode: ... def flyte_entity_call_handler( diff --git a/flytekit/core/reference_entity.py b/flytekit/core/reference_entity.py index 0d861db513..b54c4d67f6 100644 --- a/flytekit/core/reference_entity.py +++ b/flytekit/core/reference_entity.py @@ -37,8 +37,7 @@ def id(self) -> _identifier_model.Identifier: @property @abstractmethod - def resource_type(self) -> int: - ... + def resource_type(self) -> int: ... @dataclass diff --git a/flytekit/core/schedule.py b/flytekit/core/schedule.py index c69f8a2ad9..3a57cb6ab2 100644 --- a/flytekit/core/schedule.py +++ b/flytekit/core/schedule.py @@ -16,8 +16,7 @@ class LaunchPlanTriggerBase(Protocol): - def to_flyte_idl(self, *args, **kwargs) -> google_message.Message: - ... + def to_flyte_idl(self, *args, **kwargs) -> google_message.Message: ... # Duplicates flytekit.common.schedules.Schedule to avoid using the ExtendedSdkType metaclass. diff --git a/flytekit/core/task.py b/flytekit/core/task.py index ed15d3e0af..d30947509d 100644 --- a/flytekit/core/task.py +++ b/flytekit/core/task.py @@ -117,8 +117,7 @@ def task( pod_template: Optional["PodTemplate"] = ..., pod_template_name: Optional[str] = ..., accelerator: Optional[BaseAccelerator] = ..., -) -> Callable[[Callable[..., FuncOut]], PythonFunctionTask[T]]: - ... +) -> Callable[[Callable[..., FuncOut]], PythonFunctionTask[T]]: ... @overload @@ -155,8 +154,7 @@ def task( pod_template: Optional["PodTemplate"] = ..., pod_template_name: Optional[str] = ..., accelerator: Optional[BaseAccelerator] = ..., -) -> Union[PythonFunctionTask[T], Callable[..., FuncOut]]: - ... +) -> Union[PythonFunctionTask[T], Callable[..., FuncOut]]: ... def task( diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 4b1d144c88..b875bf6510 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -121,8 +121,7 @@ def modify_literal_uris(lit: Literal): ) -class TypeTransformerFailedError(TypeError, AssertionError, ValueError): - ... +class TypeTransformerFailedError(TypeError, AssertionError, ValueError): ... class TypeTransformer(typing.Generic[T]): diff --git a/flytekit/core/workflow.py b/flytekit/core/workflow.py index 0f25374717..58f8157983 100644 --- a/flytekit/core/workflow.py +++ b/flytekit/core/workflow.py @@ -804,8 +804,7 @@ def workflow( interruptible: bool = ..., on_failure: Optional[Union[WorkflowBase, Task]] = ..., docs: Optional[Documentation] = ..., -) -> Callable[[Callable[..., FuncOut]], PythonFunctionWorkflow]: - ... +) -> Callable[[Callable[..., FuncOut]], PythonFunctionWorkflow]: ... @overload @@ -815,8 +814,7 @@ def workflow( interruptible: bool = ..., on_failure: Optional[Union[WorkflowBase, Task]] = ..., docs: Optional[Documentation] = ..., -) -> Union[PythonFunctionWorkflow, Callable[..., FuncOut]]: - ... +) -> Union[PythonFunctionWorkflow, Callable[..., FuncOut]]: ... def workflow( diff --git a/flytekit/extras/accelerators.py b/flytekit/extras/accelerators.py index 139237e1fb..8a9d3e56a5 100644 --- a/flytekit/extras/accelerators.py +++ b/flytekit/extras/accelerators.py @@ -93,6 +93,7 @@ def my_task() -> None: A100_80GB """ + import abc import copy from typing import ClassVar, Generic, Optional, Type, TypeVar @@ -109,8 +110,7 @@ class BaseAccelerator(abc.ABC, Generic[T]): """ @abc.abstractmethod - def to_flyte_idl(self) -> T: - ... + def to_flyte_idl(self) -> T: ... class GPUAccelerator(BaseAccelerator): diff --git a/flytekit/extras/pytorch/__init__.py b/flytekit/extras/pytorch/__init__.py index a29d8e89e6..12c507afb9 100644 --- a/flytekit/extras/pytorch/__init__.py +++ b/flytekit/extras/pytorch/__init__.py @@ -10,6 +10,7 @@ PyTorchModuleTransformer PyTorchTensorTransformer """ + from flytekit.loggers import logger # TODO: abstract this out so that there's an established pattern for registering plugins diff --git a/flytekit/extras/sklearn/__init__.py b/flytekit/extras/sklearn/__init__.py index 1d16f6080f..d22546dbe2 100644 --- a/flytekit/extras/sklearn/__init__.py +++ b/flytekit/extras/sklearn/__init__.py @@ -7,6 +7,7 @@ SklearnEstimatorTransformer """ + from flytekit.loggers import logger # TODO: abstract this out so that there's an established pattern for registering plugins diff --git a/flytekit/remote/entities.py b/flytekit/remote/entities.py index 2af0db3afb..88fc92a547 100644 --- a/flytekit/remote/entities.py +++ b/flytekit/remote/entities.py @@ -2,6 +2,7 @@ This module contains shadow entities for all Flyte entities as represented in Flyte Admin / Control Plane. The goal is to enable easy access, manipulation of these entities. """ + from __future__ import annotations from typing import Dict, List, Optional, Tuple, Union diff --git a/flytekit/remote/executions.py b/flytekit/remote/executions.py index bd5e182952..b4ecc6550d 100644 --- a/flytekit/remote/executions.py +++ b/flytekit/remote/executions.py @@ -24,13 +24,11 @@ def inputs(self) -> Optional[LiteralsResolver]: @property @abstractmethod - def error(self) -> core_execution_models.ExecutionError: - ... + def error(self) -> core_execution_models.ExecutionError: ... @property @abstractmethod - def is_done(self) -> bool: - ... + def is_done(self) -> bool: ... @property def outputs(self) -> Optional[LiteralsResolver]: diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 5e1d3fb589..a01e68ab72 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -3,6 +3,7 @@ with a Flyte backend in an interactive and programmatic way. This of this experience as kind of like the web UI but in Python object form. """ + from __future__ import annotations import asyncio diff --git a/flytekit/remote/remote_callable.py b/flytekit/remote/remote_callable.py index 5b177bf7c4..ccd979bcdc 100644 --- a/flytekit/remote/remote_callable.py +++ b/flytekit/remote/remote_callable.py @@ -18,8 +18,7 @@ def __init__(self, *args, **kwargs): @property @abstractmethod - def name(self) -> str: - ... + def name(self) -> str: ... def construct_node_metadata(self) -> NodeMetadata: """ diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index 5c50bab9a5..ca8228b8a8 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -28,8 +28,7 @@ PathType = typing.Union[str, os.PathLike] -def noop(): - ... +def noop(): ... @dataclass diff --git a/flytekit/types/file/__init__.py b/flytekit/types/file/__init__.py index 8a2fe50b6c..703a5070f8 100644 --- a/flytekit/types/file/__init__.py +++ b/flytekit/types/file/__init__.py @@ -20,6 +20,7 @@ PythonNotebook SVGImageFile """ + import typing from typing_extensions import Annotated, get_args, get_origin diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 5c47bda998..3f36c92df8 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -21,8 +21,7 @@ from flytekit.types.pickle.pickle import FlytePickleTransformer -def noop(): - ... +def noop(): ... T = typing.TypeVar("T") diff --git a/flytekit/types/schema/types.py b/flytekit/types/schema/types.py index e7fb1d6c09..349b5aaf5a 100644 --- a/flytekit/types/schema/types.py +++ b/flytekit/types/schema/types.py @@ -68,12 +68,10 @@ def column_names(self) -> typing.Optional[typing.List[str]]: return None @abstractmethod - def iter(self, **kwargs) -> typing.Generator[T, None, None]: - ... + def iter(self, **kwargs) -> typing.Generator[T, None, None]: ... @abstractmethod - def all(self, **kwargs) -> T: - ... + def all(self, **kwargs) -> T: ... class SchemaWriter(typing.Generic[T]): @@ -95,8 +93,7 @@ def column_names(self) -> typing.Optional[typing.List[str]]: return None @abstractmethod - def write(self, *dfs, **kwargs): - ... + def write(self, *dfs, **kwargs): ... class LocalIOSchemaReader(SchemaReader[T]): diff --git a/flytekit/types/structured/__init__.py b/flytekit/types/structured/__init__.py index 7c92be78b1..7dffa49eec 100644 --- a/flytekit/types/structured/__init__.py +++ b/flytekit/types/structured/__init__.py @@ -12,7 +12,6 @@ StructuredDatasetDecoder """ - from flytekit.deck.renderer import ArrowRenderer, TopFrameRenderer from flytekit.loggers import logger diff --git a/flytekit/types/structured/structured_dataset.py b/flytekit/types/structured/structured_dataset.py index b2c8f52b9b..1d2209a2b4 100644 --- a/flytekit/types/structured/structured_dataset.py +++ b/flytekit/types/structured/structured_dataset.py @@ -1,5 +1,6 @@ from __future__ import annotations +import _datetime import collections import types import typing @@ -7,7 +8,6 @@ from dataclasses import dataclass, field, is_dataclass from typing import Dict, Generator, Optional, Type, Union -import _datetime from dataclasses_json import config from fsspec.utils import get_protocol from marshmallow import fields @@ -348,8 +348,7 @@ def get_supported_types(): return _SUPPORTED_TYPES -class DuplicateHandlerError(ValueError): - ... +class DuplicateHandlerError(ValueError): ... class StructuredDatasetTransformerEngine(TypeTransformer[StructuredDataset]): @@ -861,9 +860,9 @@ def _get_dataset_type(self, t: typing.Union[Type[StructuredDataset], typing.Any] original_python_type, column_map, storage_format, pa_schema = extract_cols_and_format(t) # type: ignore # Get the column information - converted_cols: typing.List[ - StructuredDatasetType.DatasetColumn - ] = self._convert_ordered_dict_of_columns_to_list(column_map) + converted_cols: typing.List[StructuredDatasetType.DatasetColumn] = ( + self._convert_ordered_dict_of_columns_to_list(column_map) + ) return StructuredDatasetType( columns=converted_cols, diff --git a/plugins/flytekit-async-fsspec/flytekitplugins/async_fsspec/__init__.py b/plugins/flytekit-async-fsspec/flytekitplugins/async_fsspec/__init__.py index 3cc0de14e7..09f6fd5dbd 100644 --- a/plugins/flytekit-async-fsspec/flytekitplugins/async_fsspec/__init__.py +++ b/plugins/flytekit-async-fsspec/flytekitplugins/async_fsspec/__init__.py @@ -9,6 +9,7 @@ AsyncS3FileSystem """ + import fsspec from .s3fs.s3fs import AsyncS3FileSystem diff --git a/plugins/flytekit-kf-mpi/flytekitplugins/kfmpi/task.py b/plugins/flytekit-kf-mpi/flytekitplugins/kfmpi/task.py index 665e195b4b..7c8416d007 100644 --- a/plugins/flytekit-kf-mpi/flytekitplugins/kfmpi/task.py +++ b/plugins/flytekit-kf-mpi/flytekitplugins/kfmpi/task.py @@ -2,6 +2,7 @@ This Plugin adds the capability of running distributed MPI training to Flyte using backend plugins, natively on Kubernetes. It leverages `MPI Job `_ Plugin from kubeflow. """ + from dataclasses import dataclass, field from enum import Enum from typing import Any, Callable, Dict, List, Optional, Union diff --git a/plugins/flytekit-kf-mpi/tests/test_mpi_task.py b/plugins/flytekit-kf-mpi/tests/test_mpi_task.py index f2b453fcce..deec3ff385 100644 --- a/plugins/flytekit-kf-mpi/tests/test_mpi_task.py +++ b/plugins/flytekit-kf-mpi/tests/test_mpi_task.py @@ -175,8 +175,7 @@ def test_horovod_task(serialization_settings): ), ), ) - def my_horovod_task(): - ... + def my_horovod_task(): ... cmd = my_horovod_task.get_command(serialization_settings) assert "horovodrun" in cmd diff --git a/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/error_handling.py b/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/error_handling.py index f3c509207e..f1071678c4 100644 --- a/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/error_handling.py +++ b/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/error_handling.py @@ -1,4 +1,5 @@ """Handle errors in elastic training jobs.""" + import os RECOVERABLE_ERROR_FILE_NAME = "recoverable_error" diff --git a/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py b/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py index 46eb086ad0..94b575e2a9 100644 --- a/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py +++ b/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py @@ -2,6 +2,7 @@ This Plugin adds the capability of running distributed pytorch training to Flyte using backend plugins, natively on Kubernetes. It leverages `Pytorch Job `_ Plugin from kubeflow. """ + import os from dataclasses import dataclass, field from enum import Enum diff --git a/plugins/flytekit-kf-tensorflow/flytekitplugins/kftensorflow/task.py b/plugins/flytekit-kf-tensorflow/flytekitplugins/kftensorflow/task.py index 7be1f7d030..62cd482416 100644 --- a/plugins/flytekit-kf-tensorflow/flytekitplugins/kftensorflow/task.py +++ b/plugins/flytekit-kf-tensorflow/flytekitplugins/kftensorflow/task.py @@ -2,6 +2,7 @@ This Plugin adds the capability of running distributed tensorflow training to Flyte using backend plugins, natively on Kubernetes. It leverages `TF Job `_ Plugin from kubeflow. """ + from dataclasses import dataclass, field from enum import Enum from typing import Any, Callable, Dict, Optional, Union diff --git a/plugins/flytekit-modin/flytekitplugins/modin/schema.py b/plugins/flytekit-modin/flytekitplugins/modin/schema.py index f5ab78489a..0504c38746 100644 --- a/plugins/flytekit-modin/flytekitplugins/modin/schema.py +++ b/plugins/flytekit-modin/flytekitplugins/modin/schema.py @@ -61,9 +61,9 @@ class ModinPandasDataFrameTransformer(TypeTransformer[pandas.DataFrame]): Transforms ModinPandas DataFrame's to and from a Schema (typed/untyped) """ - _SUPPORTED_TYPES: typing.Dict[ - type, SchemaType.SchemaColumn.SchemaColumnType - ] = FlyteSchemaTransformer._SUPPORTED_TYPES + _SUPPORTED_TYPES: typing.Dict[type, SchemaType.SchemaColumn.SchemaColumnType] = ( + FlyteSchemaTransformer._SUPPORTED_TYPES + ) def __init__(self): super().__init__("pandas-df-transformer", pandas.DataFrame) diff --git a/plugins/flytekit-pandera/flytekitplugins/pandera/schema.py b/plugins/flytekit-pandera/flytekitplugins/pandera/schema.py index 1c589e4c0f..6fe833d836 100644 --- a/plugins/flytekit-pandera/flytekitplugins/pandera/schema.py +++ b/plugins/flytekit-pandera/flytekitplugins/pandera/schema.py @@ -16,9 +16,9 @@ class PanderaTransformer(TypeTransformer[pandera.typing.DataFrame]): - _SUPPORTED_TYPES: typing.Dict[ - type, SchemaType.SchemaColumn.SchemaColumnType - ] = FlyteSchemaTransformer._SUPPORTED_TYPES + _SUPPORTED_TYPES: typing.Dict[type, SchemaType.SchemaColumn.SchemaColumnType] = ( + FlyteSchemaTransformer._SUPPORTED_TYPES + ) def __init__(self): super().__init__("Pandera Transformer", pandera.typing.DataFrame) # type: ignore diff --git a/plugins/flytekit-pydantic/flytekitplugins/pydantic/basemodel_transformer.py b/plugins/flytekit-pydantic/flytekitplugins/pydantic/basemodel_transformer.py index 4854360a01..50552ab108 100644 --- a/plugins/flytekit-pydantic/flytekitplugins/pydantic/basemodel_transformer.py +++ b/plugins/flytekit-pydantic/flytekitplugins/pydantic/basemodel_transformer.py @@ -1,4 +1,4 @@ -"""Serializes & deserializes the pydantic basemodels """ +"""Serializes & deserializes the pydantic basemodels""" from typing import Dict, Type diff --git a/plugins/flytekit-pydantic/flytekitplugins/pydantic/serialization.py b/plugins/flytekit-pydantic/flytekitplugins/pydantic/serialization.py index dff07883bf..5951803fdc 100644 --- a/plugins/flytekit-pydantic/flytekitplugins/pydantic/serialization.py +++ b/plugins/flytekit-pydantic/flytekitplugins/pydantic/serialization.py @@ -8,6 +8,7 @@ 3. Return a literal map with the json and the flyte object store represented as a literalmap {placeholder: flyte type} """ + import uuid from typing import Any, Dict, Union, cast From 915d0eb674bac27a154a7645073cbfdcf1551d76 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 5 Jun 2024 15:14:08 -0700 Subject: [PATCH 6/7] Fix lint error in openai plugin Signed-off-by: Eduardo Apolinario --- .../flytekit-openai/flytekitplugins/openai/batch/task.py | 2 +- .../flytekitplugins/openai/batch/workflow.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/flytekit-openai/flytekitplugins/openai/batch/task.py b/plugins/flytekit-openai/flytekitplugins/openai/batch/task.py index f5054582dd..695e8882e6 100644 --- a/plugins/flytekit-openai/flytekitplugins/openai/batch/task.py +++ b/plugins/flytekit-openai/flytekitplugins/openai/batch/task.py @@ -33,7 +33,7 @@ def __init__( self, name: str, openai_organization: str, - config: Dict[str, Any] = {}, + config: Dict[str, Any], **kwargs, ): super().__init__( diff --git a/plugins/flytekit-openai/flytekitplugins/openai/batch/workflow.py b/plugins/flytekit-openai/flytekitplugins/openai/batch/workflow.py index 1f0ff30b51..209bd0d981 100644 --- a/plugins/flytekit-openai/flytekitplugins/openai/batch/workflow.py +++ b/plugins/flytekit-openai/flytekitplugins/openai/batch/workflow.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Iterator +from typing import Any, Dict, Iterator, Optional from flytekit import Resources, Workflow from flytekit.models.security import Secret @@ -18,7 +18,7 @@ def create_batch( name: str, openai_organization: str, secret: Secret, - config: Dict[str, Any] = {}, + config: Optional[Dict[str, Any]] = None, is_json_iterator: bool = True, file_upload_mem: str = "700Mi", file_download_mem: str = "700Mi", @@ -45,6 +45,8 @@ def create_batch( name=f"openai-file-upload-{name.replace('.', '')}", task_config=OpenAIFileConfig(openai_organization=openai_organization, secret=secret), ) + if config is None: + config = {} batch_endpoint_task_obj = BatchEndpointTask( name=f"openai-batch-{name.replace('.', '')}", openai_organization=openai_organization, From 41ab1ccaa9c1bd322c5a170a2fd269d98a2a86e3 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 5 Jun 2024 15:37:04 -0700 Subject: [PATCH 7/7] Handle nil databricks_conf in spark plugin Signed-off-by: Eduardo Apolinario --- plugins/flytekit-spark/flytekitplugins/spark/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/flytekit-spark/flytekitplugins/spark/models.py b/plugins/flytekit-spark/flytekitplugins/spark/models.py index ac03f3a7cd..e74a9fbe3f 100644 --- a/plugins/flytekit-spark/flytekitplugins/spark/models.py +++ b/plugins/flytekit-spark/flytekitplugins/spark/models.py @@ -25,7 +25,7 @@ def __init__( spark_conf: Dict[str, str], hadoop_conf: Dict[str, str], executor_path: str, - databricks_conf: Dict[str, Dict[str, Dict]], + databricks_conf: Optional[Dict[str, Dict[str, Dict]]] = None, databricks_instance: Optional[str] = None, ): """ @@ -43,6 +43,8 @@ def __init__( self._executor_path = executor_path self._spark_conf = spark_conf self._hadoop_conf = hadoop_conf + if databricks_conf is None: + databricks_conf = {} self._databricks_conf = databricks_conf self._databricks_instance = databricks_instance