Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Generate unique SageMaker training job name based on pipeline and ste… #1505

Merged
merged 7 commits into from
May 2, 2023
8 changes: 5 additions & 3 deletions examples/airflow_orchestration/steps/importers.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@


@step
def importer_mnist() -> Output(
train_dataloader=DataLoader,
test_dataloader=DataLoader,
def importer_mnist() -> (
Output(
train_dataloader=DataLoader,
test_dataloader=DataLoader,
)
):
"""Download the Fashion MNIST dataset."""
# Download training data from open datasets.
Expand Down
8 changes: 5 additions & 3 deletions examples/bentoml_deployment/steps/importers.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@


@step
def importer_mnist() -> Output(
train_dataloader=DataLoader,
test_dataloader=DataLoader,
def importer_mnist() -> (
Output(
train_dataloader=DataLoader,
test_dataloader=DataLoader,
)
):
"""Download the Fashion MNIST dataset."""
# Download training data from open datasets.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@


@step
def tf_data_loader() -> Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
def tf_data_loader() -> (
Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
)
):
"""Download the MNIST data store it as an artifact."""
(x_train, y_train), (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@


@step
def tf_data_loader() -> Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
def tf_data_loader() -> (
Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
)
):
"""Download the MNIST data store it as an artifact."""
(x_train, y_train), (
Expand Down
4 changes: 2 additions & 2 deletions examples/deepchecks_data_validation/steps/data_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@


@step
def data_loader() -> Output(
reference_dataset=pd.DataFrame, comparison_dataset=pd.DataFrame
def data_loader() -> (
Output(reference_dataset=pd.DataFrame, comparison_dataset=pd.DataFrame)
):
"""Load the iris dataset."""
iris_df = iris.load_data(data_format="Dataframe", as_train_test=False)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ class MNISTDigitClassifier(ImageClassifier):
"""
MNISTDigitClassifier handler class. This handler extends class ImageClassifier from image_classifier.py, a
default handler. This handler takes an image and returns the number in that image.
Here the post-processing method has been overridden while others are reused from parent class."""
Here the post-processing method has been overridden while others are reused from parent class.
"""

image_processing = transforms.Compose(
[transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@


@step
def importer_mnist() -> Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
def importer_mnist() -> (
Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
)
):
"""Download the MNIST data and store it as an artifact."""
(x_train, y_train), (
Expand Down
12 changes: 7 additions & 5 deletions examples/kubeflow_pipelines_orchestration/steps/importers.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@


@step
def importer() -> Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
def importer() -> (
Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
)
):
"""Download the MNIST data store it as an artifact."""
(X_train, y_train), (
Expand Down
12 changes: 7 additions & 5 deletions examples/kubernetes_orchestration/steps/importer.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@


@step
def digits_data_loader() -> Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
def digits_data_loader() -> (
Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
)
):
"""Loads the digits dataset as a tuple of flattened numpy arrays."""
digits = load_digits()
Expand Down
12 changes: 7 additions & 5 deletions examples/mlflow_tracking/steps/loader/loader_step.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@


@step
def loader_mnist() -> Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
def loader_mnist() -> (
Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
)
):
"""Download the MNIST data store it as an artifact."""
(x_train, y_train), (
Expand Down
12 changes: 7 additions & 5 deletions examples/neptune_tracking/steps/loader/loader_step.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@


@step
def loader_mnist() -> Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
def loader_mnist() -> (
Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
)
):
"""Download the MNIST data store it as an artifact."""
(x_train, y_train), (
Expand Down
8 changes: 5 additions & 3 deletions examples/pytorch/steps/importers.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@


@step
def importer_mnist() -> Output(
train_dataloader=DataLoader,
test_dataloader=DataLoader,
def importer_mnist() -> (
Output(
train_dataloader=DataLoader,
test_dataloader=DataLoader,
)
):
"""Download the Fashion MNIST dataset."""
# Download training data from open datasets.
Expand Down
1 change: 0 additions & 1 deletion examples/quickstart/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@


def main():

# initialize and run the training pipeline
training_pipeline_instance = training_pipeline(
training_data_loader=training_data_loader(),
Expand Down
12 changes: 7 additions & 5 deletions examples/quickstart/steps/data_loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@


@step
def training_data_loader() -> Output(
X_train=pd.DataFrame,
X_test=pd.DataFrame,
y_train=pd.Series,
y_test=pd.Series,
def training_data_loader() -> (
Output(
X_train=pd.DataFrame,
X_test=pd.DataFrame,
y_train=pd.Series,
y_test=pd.Series,
)
):
"""Load the iris dataset as tuple of Pandas DataFrame / Series."""
iris = load_iris(as_frame=True)
Expand Down
1 change: 0 additions & 1 deletion examples/quickstart_py37/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@


def main():

# initialize and run the training pipeline
training_pipeline_instance = training_pipeline(
training_data_loader=training_data_loader(),
Expand Down
12 changes: 7 additions & 5 deletions examples/quickstart_py37/steps/data_loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@


@step
def training_data_loader() -> Output(
X_train=pd.DataFrame,
X_test=pd.DataFrame,
y_train=pd.Series,
y_test=pd.Series,
def training_data_loader() -> (
Output(
X_train=pd.DataFrame,
X_test=pd.DataFrame,
y_train=pd.Series,
y_test=pd.Series,
)
):
"""Load the iris dataset as tuple of Pandas DataFrame / Series."""
iris = load_iris(as_frame=True)
Expand Down
8 changes: 5 additions & 3 deletions examples/sagemaker_orchestration/steps/importers.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@


@step
def importer_mnist() -> Output(
train_dataloader=DataLoader,
test_dataloader=DataLoader,
def importer_mnist() -> (
Output(
train_dataloader=DataLoader,
test_dataloader=DataLoader,
)
):
"""Download the Fashion MNIST dataset."""
# Download training data from open datasets.
Expand Down
12 changes: 7 additions & 5 deletions examples/scipy/steps/loader/loader_step.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,13 @@ def clean_text(text: str):


@step
def importer() -> Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
def importer() -> (
Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
)
):
train = pd.read_csv(TRAIN_PATH)
test = pd.read_csv(TEST_PATH)
Expand Down
12 changes: 7 additions & 5 deletions examples/seldon_deployment/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,13 @@


@step
def importer_mnist() -> Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
def importer_mnist() -> (
Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
)
):
"""Download the MNIST data store it as an artifact."""
(x_train, y_train), (
Expand Down
12 changes: 7 additions & 5 deletions examples/slack_alert/steps/data_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@


@step
def digits_data_loader() -> Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
def digits_data_loader() -> (
Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
)
):
"""Loads the digits dataset as a tuple of flattened numpy arrays."""
digits = load_digits()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@


@step
def digits_data_loader() -> Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
def digits_data_loader() -> (
Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
)
):
"""Loads the digits dataset as a tuple of flattened numpy arrays."""
digits = load_digits()
Expand Down
12 changes: 7 additions & 5 deletions examples/tekton_pipelines_orchestration/steps/importers.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@


@step
def importer() -> Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
def importer() -> (
Output(
X_train=np.ndarray,
X_test=np.ndarray,
y_train=np.ndarray,
y_test=np.ndarray,
)
):
"""Download the MNIST data store it as an artifact."""
(X_train, y_train), (
Expand Down
1 change: 0 additions & 1 deletion examples/wandb_tracking/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
from steps.trainer.trainer_step import TrainerParameters, tf_trainer

if __name__ == "__main__":

# Initialize a pipeline run
run_1 = wandb_example_pipeline(
importer=importer_mnist(),
Expand Down
12 changes: 7 additions & 5 deletions examples/wandb_tracking/steps/importer/importer_step.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@


@step
def importer_mnist() -> Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
def importer_mnist() -> (
Output(
x_train=np.ndarray,
y_train=np.ndarray,
x_test=np.ndarray,
y_test=np.ndarray,
)
):
"""Download the MNIST data store it as an artifact."""
(x_train, y_train), (
Expand Down
1 change: 0 additions & 1 deletion examples/whylogs_data_profiling/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ def visualize_statistics(


if __name__ == "__main__":

pipeline_instance = data_profiling_pipeline(
data_loader=data_loader(),
data_splitter=data_splitter(),
Expand Down
Loading