From 7d763ab8fd7c4f4d6f891ac3a49d818b189a440a Mon Sep 17 00:00:00 2001 From: Mohamed Shaban Date: Wed, 3 Mar 2021 11:42:30 -0500 Subject: [PATCH] samples hero scenarios (#16936) * [samples] added 'batch_translation_async' sample * [samples] added 'batch_translation_with_storage_async' sample * [samples] added remianing async samples * [samples] update file names * [samples] added self to instance methods * [samples][async] fix import textanalytics :) * [samples] fix self. when calling instance methods * [samples] fixed async check status to use AsyncItemPaged used in Async Client * [samples] async -> some async operations instead of sync ones * [samples][async] use async blob operations * [samples][async] blob download async * [samples][async] check_documents async * [samples][async] added some missing await methods * [async samples] change await time to recommended period * [samples] updated async samples to comply with new changes --- .../samples/sample_azure_storage.py | 122 ------------------ .../samples/sample_check_statuses.py | 88 ------------- .../samples/sample_list_batches.py | 41 ------ 3 files changed, 251 deletions(-) delete mode 100644 sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_azure_storage.py delete mode 100644 sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_check_statuses.py delete mode 100644 sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_list_batches.py diff --git a/sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_azure_storage.py b/sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_azure_storage.py deleted file mode 100644 index dcbb9ec80298..000000000000 --- a/sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_azure_storage.py +++ /dev/null @@ -1,122 +0,0 @@ -# coding=utf-8 -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - - -def batch_translation_with_storage(): - import os - from azure.core.credentials import AzureKeyCredential - from azure.ai.documenttranslation import ( - DocumentTranslationClient, - BatchDocumentInput, - StorageTarget - ) - from azure.storage.blob import ContainerClient, generate_container_sas, ContainerSasPermissions - - endpoint = os.environ["AZURE_DOCUMENT_TRANSLATION_ENDPOINT"] - key = os.environ["AZURE_DOCUMENT_TRANSLATION_KEY"] - source_storage_endpoint = os.environ["AZURE_STORAGE_SOURCE_ENDPOINT"] - source_storage_account_name = os.environ["AZURE_STORAGE_SOURCE_ACCOUNT_NAME"] - source_storage_container_name = os.environ["AZURE_STORAGE_SOURCE_CONTAINER_NAME"] - source_storage_key = os.environ["AZURE_STORAGE_SOURCE_KEY"] - target_storage_endpoint = os.environ["AZURE_STORAGE_TARGET_ENDPOINT"] - target_storage_account_name = os.environ["AZURE_STORAGE_TARGET_ACCOUNT_NAME"] - target_storage_container_name = os.environ["AZURE_STORAGE_TARGET_CONTAINER_NAME"] - target_storage_key = os.environ["AZURE_STORAGE_TARGET_KEY"] - - translation_client = DocumentTranslationClient( - endpoint, AzureKeyCredential(key) - ) - - container_client = ContainerClient( - source_storage_endpoint, - container_name=source_storage_container_name, - credential=source_storage_key - ) - - with open("document.txt", "rb") as doc: - container_client.upload_blob("document.txt", doc) - - source_container_sas = generate_container_sas( - account_name=source_storage_account_name, - container_name=source_storage_container_name, - account_key=source_storage_key, - permission=ContainerSasPermissions.from_string("rl") - ) - - target_container_sas = generate_container_sas( - account_name=target_storage_account_name, - container_name=target_storage_container_name, - account_key=target_storage_key, - permission=ContainerSasPermissions.from_string("rlwd") - ) - - source_container_url = source_storage_endpoint + "/" + source_storage_container_name + "?" + source_container_sas - target_container_url = target_storage_endpoint + "/" + target_storage_container_name + "?" + target_container_sas - - batch = [ - BatchDocumentInput( - source_url=source_container_url, - targets=[ - StorageTarget( - target_url=target_container_url, - language="es" - ) - ], - prefix="document" - ) - ] - - job_detail = translation_client.create_translation_job(batch) - job_result = translation_client.wait_until_done(job_detail.id) - - if job_result.status == "Succeeded": - print("We translated our documents!") - if job_result.documents_failed_count > 0: - check_documents(translation_client, job_result.id) - - elif job_result.status in ["Failed", "ValidationFailed"]: - if job_result.error: - print("Translation job failed: {}: {}".format(job_result.error.code, job_result.error.message)) - check_documents(translation_client, job_result.id) - exit(1) - - container_client = ContainerClient( - target_storage_endpoint, - container_name=target_storage_container_name, - credential=target_storage_key - ) - - target_container_client = container_client.from_container_url(target_container_url) - - with open("translated.txt", "wb") as my_blob: - download_stream = target_container_client.download_blob("document.txt") - my_blob.write(download_stream.readall()) - - -def check_documents(client, job_id): - from azure.core.exceptions import ResourceNotFoundError - - try: - doc_statuses = client.list_documents_statuses(job_id) # type: ItemPaged[DocumentStatusDetail] - except ResourceNotFoundError as err: - print("Failed to process any documents in source/target container.") - raise err - - docs_to_retry = [] - for document in doc_statuses: - if document.status == "Failed": - print("Document at {} failed to be translated to {} language".format( - document.url, document.translate_to - )) - print("Document ID: {}, Error Code: {}, Message: {}".format( - document.id, document.error.code, document.error.message - )) - if document.url not in docs_to_retry: - docs_to_retry.append(document.url) - - -if __name__ == '__main__': - batch_translation_with_storage() diff --git a/sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_check_statuses.py b/sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_check_statuses.py deleted file mode 100644 index 596b373db915..000000000000 --- a/sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_check_statuses.py +++ /dev/null @@ -1,88 +0,0 @@ -# coding=utf-8 -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - - -def sample_translation_status_checks(): - import os - import time - from azure.core.credentials import AzureKeyCredential - from azure.ai.documenttranslation import ( - DocumentTranslationClient, - BatchDocumentInput, - StorageTarget - ) - - endpoint = os.environ["AZURE_DOCUMENT_TRANSLATION_ENDPOINT"] - key = os.environ["AZURE_DOCUMENT_TRANSLATION_KEY"] - source_container_url = os.environ["AZURE_SOURCE_CONTAINER_URL"] - target_container_url_es = os.environ["AZURE_TARGET_CONTAINER_URL_ES"] - target_container_url_fr = os.environ["AZURE_TARGET_CONTAINER_URL_FR"] - - client = DocumentTranslationClient(endpoint, AzureKeyCredential(key)) - - batch = [ - BatchDocumentInput( - source_url=source_container_url, - targets=[ - StorageTarget( - target_url=target_container_url_es, - language="es" - ), - StorageTarget( - target_url=target_container_url_fr, - language="fr" - ) - ], - storage_type="folder", - prefix="document_2021" - ) - ] - - job_detail = client.create_translation_job(batch) - - while True: - job_detail = client.get_job_status(job_detail.id) # type: JobStatusDetail - if job_detail.status in ["NotStarted", "Running"]: - time.sleep(30) - continue - - elif job_detail.status in ["Failed", "ValidationFailed"]: - if job_detail.error: - print("Translation job failed: {}: {}".format(job_detail.error.code, job_detail.error.message)) - check_documents(client, job_detail.id) - exit(1) - - elif job_detail.status == "Succeeded": - print("We translated our documents!") - if job_detail.documents_failed_count > 0: - check_documents(client, job_detail.id) - break - - -def check_documents(client, job_id): - from azure.core.exceptions import ResourceNotFoundError - - try: - doc_statuses = client.list_documents_statuses(job_id) # type: ItemPaged[DocumentStatusDetail] - except ResourceNotFoundError as err: - print("Failed to process any documents in source/target container due to insufficient permissions.") - raise err - - docs_to_retry = [] - for document in doc_statuses: - if document.status == "Failed": - print("Document at {} failed to be translated to {} language".format( - document.url, document.translate_to - )) - print("Document ID: {}, Error Code: {}, Message: {}".format( - document.id, document.error.code, document.error.message - )) - if document.url not in docs_to_retry: - docs_to_retry.append(document.url) - - -if __name__ == '__main__': - sample_translation_status_checks() diff --git a/sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_list_batches.py b/sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_list_batches.py deleted file mode 100644 index 2961b9516301..000000000000 --- a/sdk/documenttranslation/azure-ai-documenttranslation/samples/sample_list_batches.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding=utf-8 -# ------------------------------------ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# ------------------------------------ - - -def sample_list_all_jobs(): - import os - from azure.core.credentials import AzureKeyCredential - from azure.ai.documenttranslation import ( - DocumentTranslationClient, - ) - - endpoint = os.environ["AZURE_DOCUMENT_TRANSLATION_ENDPOINT"] - key = os.environ["AZURE_DOCUMENT_TRANSLATION_KEY"] - - client = DocumentTranslationClient(endpoint, AzureKeyCredential(key)) - jobs = client.list_submitted_jobs() # type: ItemPaged[JobStatusDetail] - - for job in jobs: - if job.status in ["NotStarted", "Running"]: - job = client.wait_until_done(job.id) - - print("Job ID: {}".format(job.id)) - print("Job status: {}".format(job.status)) - print("Job created on: {}".format(job.created_on)) - print("Job last updated on: {}".format(job.last_updated_on)) - print("Total number of translations on documents: {}".format(job.documents_total_count)) - print("Total number of characters charged: {}".format(job.total_characters_charged)) - - print("Of total documents...") - print("{} failed".format(job.documents_failed_count)) - print("{} succeeded".format(job.documents_succeeded_count)) - print("{} in progress".format(job.documents_in_progress_count)) - print("{} not yet started".format(job.documents_not_yet_started_count)) - print("{} cancelled".format(job.documents_cancelled_count)) - - -if __name__ == '__main__': - sample_list_all_jobs()