From 7e89995cc8e50dbebe7f9750bfbc9156516075e2 Mon Sep 17 00:00:00 2001 From: Praven Kuttappan <55455725+praveenkuttappan@users.noreply.github.com> Date: Wed, 7 Aug 2024 10:12:00 -0400 Subject: [PATCH] Update apiview prod to staging sync job to use managed identity (#8673) * Update apiview prod to staging sync job to use managed identity * Update job name * Add python install step and update as per review comment * Connect to cosmos DB using key * apiview_sync_pipeline * Changes to return exit code * Add new line * Add chain credential to support cli credential --- .../python/apiview-syncdb/requirements.txt | 3 ++- .../python/apiview-syncdb/sync_cosmosdb.py | 27 +++++++++---------- src/dotnet/APIView/apiview-sync-staging.yml | 24 +++++++++++------ 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/eng/scripts/python/apiview-syncdb/requirements.txt b/eng/scripts/python/apiview-syncdb/requirements.txt index dda0291d7b8..35697b67285 100644 --- a/eng/scripts/python/apiview-syncdb/requirements.txt +++ b/eng/scripts/python/apiview-syncdb/requirements.txt @@ -1,2 +1,3 @@ azure-cosmos -azure-storage-blob \ No newline at end of file +azure-storage-blob +azure-identity \ No newline at end of file diff --git a/eng/scripts/python/apiview-syncdb/sync_cosmosdb.py b/eng/scripts/python/apiview-syncdb/sync_cosmosdb.py index 3d43a09e02e..4feaf3cbcc9 100644 --- a/eng/scripts/python/apiview-syncdb/sync_cosmosdb.py +++ b/eng/scripts/python/apiview-syncdb/sync_cosmosdb.py @@ -8,6 +8,7 @@ import traceback from ast import literal_eval from azure.cosmos import CosmosClient +from azure.identity import AzurePowerShellCredential, ChainedTokenCredential, AzureCliCredential from azure.storage.blob import BlobServiceClient logging.getLogger().setLevel(logging.INFO) @@ -25,13 +26,14 @@ BACKUP_CONTAINER = "backups" BLOB_NAME_PATTERN ="cosmos/{0}/{1}" +# Create a AzurePowerShellCredential() +credential_chain = ChainedTokenCredential(AzureCliCredential(), AzurePowerShellCredential()) +def restore_data_from_backup(backup_storage_url, dest_url, db_name): -def restore_data_from_backup(connection_string, dest_url, dest_key, db_name): - - dest_db_client = get_db_client(dest_url, dest_key, db_name) + dest_db_client = get_db_client(dest_url, db_name) - blob_service_client = BlobServiceClient.from_connection_string(connection_string) + blob_service_client = BlobServiceClient(backup_storage_url, credential = credential_chain) container_client = blob_service_client.get_container_client(BACKUP_CONTAINER) for cosmos_container_name in COSMOS_CONTAINERS: # Load source records from backup file @@ -68,10 +70,10 @@ def get_backup_contents(container_client, blob_name): # Create cosmosdb clients -def get_db_client(dest_url, dest_key, db_name): +def get_db_client(dest_url, db_name): # Create cosmosdb client for destination db - dest_cosmos_client = CosmosClient(dest_url, credential=dest_key) + dest_cosmos_client = CosmosClient(dest_url, credential=credential_chain) if not dest_cosmos_client: logging.error("Failed to create cosmos client for destination db") exit(1) @@ -83,7 +85,7 @@ def get_db_client(dest_url, dest_key, db_name): dest_db_client = dest_cosmos_client.get_database_client(db_name) logging.info("Created database clients") except: - logging.error("Failed to create databae client using CosmosClient") + logging.error("Failed to create database client using CosmosClient") traceback.print_exc() exit(1) return dest_db_client @@ -122,20 +124,15 @@ def fetch_records(container_client, container_name): ) parser.add_argument( - "--backup-connection-string", + "--backup-storage-url", required=True, - help=("Connection string to backup storage account"), + help=("URL to backup storage account"), ) parser.add_argument( "--dest-url", required=True, help=("URL to destination cosmosdb"), ) - parser.add_argument( - "--dest-key", - required=True, - help=("Destination cosmosdb account key"), - ) parser.add_argument( "--db-name", required=True, @@ -145,4 +142,4 @@ def fetch_records(container_client, container_name): args = parser.parse_args() logging.info("Syncing database..") - restore_data_from_backup(args.backup_connection_string, args.dest_url, args.dest_key, args.db_name) \ No newline at end of file + restore_data_from_backup(args.backup_storage_url, args.dest_url, args.db_name) \ No newline at end of file diff --git a/src/dotnet/APIView/apiview-sync-staging.yml b/src/dotnet/APIView/apiview-sync-staging.yml index 9ab4033a893..22ac88e43e6 100644 --- a/src/dotnet/APIView/apiview-sync-staging.yml +++ b/src/dotnet/APIView/apiview-sync-staging.yml @@ -12,6 +12,7 @@ stages: - stage: 'Main' jobs: - job: 'BlobCopy' + displayName: 'Sync API reviews' pool: name: azsdk-pool-mms-win-2022-general vmImage: windows-2022 @@ -40,13 +41,20 @@ stages: env: AZCOPY_AUTO_LOGIN_TYPE: 'PSCRED' - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6' - inputs: - versionSpec: 3.6 + - template: /eng/pipelines/templates/steps/use-python-version.yml + parameters: + versionSpec: '3.10' - - script: | - cd $(Build.SourcesDirectory)/eng/scripts/python/apiview-syncdb/ - python -m pip install -r requirements.txt - python ./sync_cosmosdb.py --dest-url $(apiview-staging-cosmos-url) --dest-key $(apiview-staging-cosmos-key) --db-name $(apiview-cosmosdb-name) --backup-connection-string $(apiview-cosmos-backup-connection) + - task: AzurePowerShell@5 displayName: Sync CosmosDB + inputs: + azureSubscription: 'Azure SDK Engineering System' + ScriptType: 'InlineScript' + azurePowerShellVersion: LatestVersion + workingDirectory: $(Build.BinariesDirectory) + pwsh: true + Inline: | + cd $(Build.SourcesDirectory)/eng/scripts/python/apiview-syncdb/ + python -m pip install -r requirements.txt + python ./sync_cosmosdb.py --dest-url $(apiview-staging-cosmos-url) --db-name $(apiview-cosmosdb-name) --backup-storage-url $(apiview-cosmos-backup-url) + exit $LASTEXITCODE