From 374b8bbcc0335bb06444748f427d3ce67751df3f Mon Sep 17 00:00:00 2001 From: Noah Negrey Date: Thu, 12 Mar 2020 13:56:04 -0600 Subject: [PATCH] automl: move samples to beta set (#3045) --- automl/beta/get_model_evaluation.py | 11 +++++ automl/beta/get_operation_status.py | 34 ++++++++++++++++ automl/beta/get_operation_status_test.py | 40 ++++++++++++++++++ automl/beta/import_dataset.py | 39 ++++++++++++++++++ automl/beta/import_dataset_test.py | 41 +++++++++++++++++++ automl/beta/list_datasets.py | 52 ++++++++++++++++++++++++ automl/beta/list_datasets_test.py | 27 ++++++++++++ 7 files changed, 244 insertions(+) create mode 100644 automl/beta/get_operation_status.py create mode 100644 automl/beta/get_operation_status_test.py create mode 100644 automl/beta/import_dataset.py create mode 100644 automl/beta/import_dataset_test.py create mode 100644 automl/beta/list_datasets.py create mode 100644 automl/beta/list_datasets_test.py diff --git a/automl/beta/get_model_evaluation.py b/automl/beta/get_model_evaluation.py index 147547c06d83..ed540f2e1c33 100644 --- a/automl/beta/get_model_evaluation.py +++ b/automl/beta/get_model_evaluation.py @@ -16,6 +16,7 @@ def get_model_evaluation(project_id, model_id, model_evaluation_id): """Get model evaluation.""" # [START automl_video_classification_get_model_evaluation_beta] + # [START automl_video_object_tracking_get_model_evaluation_beta] from google.cloud import automl_v1beta1 as automl # TODO(developer): Uncomment and set the following variables @@ -41,9 +42,19 @@ def get_model_evaluation(project_id, model_id, model_evaluation_id): "Evaluation example count: {}".format(response.evaluated_example_count) ) + # [END automl_video_object_tracking_get_model_evaluation_beta] + print( "Classification model evaluation metrics: {}".format( response.classification_evaluation_metrics ) ) # [END automl_video_classification_get_model_evaluation_beta] + + # [START automl_video_object_tracking_get_model_evaluation_beta] + print( + "Video object tracking model evaluation metrics: {}".format( + response.video_object_tracking_evaluation_metrics + ) + ) + # [END automl_video_object_tracking_get_model_evaluation_beta] diff --git a/automl/beta/get_operation_status.py b/automl/beta/get_operation_status.py new file mode 100644 index 000000000000..f376e2461e4e --- /dev/null +++ b/automl/beta/get_operation_status.py @@ -0,0 +1,34 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START automl_get_operation_status_beta] +from google.cloud import automl_v1beta1 as automl + + +def get_operation_status( + operation_full_id="projects/YOUR_PROJECT_ID/locations/us-central1/" + "operations/YOUR_OPERATION_ID", +): + """Get operation status.""" + client = automl.AutoMlClient() + + # Get the latest state of a long-running operation. + response = client.transport._operations_client.get_operation( + operation_full_id + ) + + print("Name: {}".format(response.name)) + print("Operation details:") + print(response) +# [END automl_get_operation_status_beta] diff --git a/automl/beta/get_operation_status_test.py b/automl/beta/get_operation_status_test.py new file mode 100644 index 000000000000..7da9e7b3b0ef --- /dev/null +++ b/automl/beta/get_operation_status_test.py @@ -0,0 +1,40 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud import automl_v1beta1 as automl +import pytest + +import get_operation_status + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] + + +@pytest.fixture(scope="function") +def operation_id(): + client = automl.AutoMlClient() + project_location = client.location_path(PROJECT_ID, "us-central1") + generator = client.transport._operations_client.list_operations( + project_location, filter_="" + ).pages + page = next(generator) + operation = page.next() + yield operation.name + + +def test_get_operation_status(capsys, operation_id): + get_operation_status.get_operation_status(operation_id) + out, _ = capsys.readouterr() + assert "Operation details" in out diff --git a/automl/beta/import_dataset.py b/automl/beta/import_dataset.py new file mode 100644 index 000000000000..4a1f1d926f93 --- /dev/null +++ b/automl/beta/import_dataset.py @@ -0,0 +1,39 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START automl_import_data_beta] +from google.cloud import automl_v1beta1 as automl + + +def import_dataset( + project_id="YOUR_PROJECT_ID", + dataset_id="YOUR_DATASET_ID", + path="gs://YOUR_BUCKET_ID/path/to/data.csv", +): + """Import a dataset.""" + client = automl.AutoMlClient() + # Get the full path of the dataset. + dataset_full_id = client.dataset_path( + project_id, "us-central1", dataset_id + ) + # Get the multiple Google Cloud Storage URIs + input_uris = path.split(",") + gcs_source = automl.types.GcsSource(input_uris=input_uris) + input_config = automl.types.InputConfig(gcs_source=gcs_source) + # Import data from the input URI + response = client.import_data(dataset_full_id, input_config) + + print("Processing import...") + print("Data imported. {}".format(response.result())) +# [END automl_import_data_beta] diff --git a/automl/beta/import_dataset_test.py b/automl/beta/import_dataset_test.py new file mode 100644 index 000000000000..35d23edc7e8f --- /dev/null +++ b/automl/beta/import_dataset_test.py @@ -0,0 +1,41 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import import_dataset + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +BUCKET_ID = "{}-lcm".format(PROJECT_ID) +DATASET_ID = "TEN0000000000000000000" + + +def test_import_dataset(capsys): + # As importing a dataset can take a long time and only four operations can + # be run on a dataset at once. Try to import into a nonexistent dataset and + # confirm that the dataset was not found, but other elements of the request + # were valid. + try: + data = "gs://{}/sentiment-analysis/dataset.csv".format(BUCKET_ID) + import_dataset.import_dataset(PROJECT_ID, DATASET_ID, data) + out, _ = capsys.readouterr() + assert ( + "The Dataset doesn't exist or is inaccessible for use with AutoMl." + in out + ) + except Exception as e: + assert ( + "The Dataset doesn't exist or is inaccessible for use with AutoMl." + in e.message + ) diff --git a/automl/beta/list_datasets.py b/automl/beta/list_datasets.py new file mode 100644 index 000000000000..1fe490402bd0 --- /dev/null +++ b/automl/beta/list_datasets.py @@ -0,0 +1,52 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START automl_video_classification_list_datasets_beta] +# [START automl_video_object_tracking_list_datasets_beta] +from google.cloud import automl_v1beta1 as automl + + +def list_datasets(project_id="YOUR_PROJECT_ID"): + """List datasets.""" + client = automl.AutoMlClient() + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, "us-central1") + + # List all the datasets available in the region. + response = client.list_datasets(project_location, "") + + print("List of datasets:") + for dataset in response: + print("Dataset name: {}".format(dataset.name)) + print("Dataset id: {}".format(dataset.name.split("/")[-1])) + print("Dataset display name: {}".format(dataset.display_name)) + print("Dataset create time:") + print("\tseconds: {}".format(dataset.create_time.seconds)) + print("\tnanos: {}".format(dataset.create_time.nanos)) + # [END automl_video_object_tracking_list_datasets_beta] + + print( + "Video classification dataset metadata: {}".format( + dataset.video_classification_dataset_metadata + ) + ) + # [END automl_video_classification_list_datasets_beta] + + # [START automl_video_object_tracking_list_datasets_beta] + print( + "Video object tracking dataset metadata: {}".format( + dataset.video_object_tracking_dataset_metadata + ) + ) + # [END automl_video_object_tracking_list_datasets_beta] diff --git a/automl/beta/list_datasets_test.py b/automl/beta/list_datasets_test.py new file mode 100644 index 000000000000..7057af815d19 --- /dev/null +++ b/automl/beta/list_datasets_test.py @@ -0,0 +1,27 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import list_datasets + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +DATASET_ID = os.environ["ENTITY_EXTRACTION_DATASET_ID"] + + +def test_list_dataset(capsys): + # list datasets + list_datasets.list_datasets(PROJECT_ID) + out, _ = capsys.readouterr() + assert "Dataset id: {}".format(DATASET_ID) in out