From 29b300b03b5366a3b2a9acfdc3491a5f0377f426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20C=C3=A1rdenas?= Date: Thu, 22 Feb 2024 16:22:59 -0800 Subject: [PATCH] Fix errors in repo structure cache script and improve pipeline (#7733) * Update repo URLs and requirements file path for cache job * Fixing bug in script * Update yaml to match PPE and PROD database names and repo branches * Use git owner uppercase to match DB key in PPE and PROD * Update both PPE and PROD in different steps using the same pipeline --- .../openapitools/repo-structure-cache-job.yml | 34 ++++++++----------- .../sync-repo-structure-cache.py | 6 ++-- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/eng/pipelines/openapitools/repo-structure-cache-job.yml b/eng/pipelines/openapitools/repo-structure-cache-job.yml index 392b72690c2..debcb2075da 100644 --- a/eng/pipelines/openapitools/repo-structure-cache-job.yml +++ b/eng/pipelines/openapitools/repo-structure-cache-job.yml @@ -1,30 +1,26 @@ -parameters: -- name: DB_NAME - type: string - default: "openapi-portal" - -- name: COLLECTION_NAME - type: string - default: "repoStructureCache" - -- name: REPOS_URL_LIST - type: string - default: "https://github.com/test-repo-billy/azure-rest-api-specs/tree/main,https://github.com/test-repo-billy/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/test-repo-billy/azure-rest-api-specs-pr/tree/main,https://github.com/test-repo-billy/azure-rest-api-specs/tree/RPSaaSDev" - -trigger: -- main +trigger: none +pr: none pool: name: azsdk-pool-mms-ubuntu-2204-general vmImage: ubuntu-22.04 steps: - - script: | - pip install -r 'eng/scripts/OpenAPI Hub/requirements.txt' + pip install -r 'eng/scripts/openapitools/python/cacherepostructure/requirements.txt' pip list - python 'eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py' --db_name ${{ parameters.DB_NAME }} --collection_name ${{ parameters.COLLECTION_NAME }} --repos_url_list ${{ parameters.REPOS_URL_LIST }} --repo_clone_path $(System.DefaultWorkingDirectory) - displayName: Run Script + displayName: Install dependencies + +- script: | + python 'eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py' --db_name "openapiPortal" --collection_name "repoStructureCache" --repos_url_list "https://github.com/Azure/azure-rest-api-specs/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/Azure/azure-rest-api-specs-pr/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSMaster" --repo_clone_path $(System.DefaultWorkingDirectory) + displayName: Update PROD cache env: MONGO_CONNECTION_STRING: $(Open-API-MongoDB-Connection-String) + GITHUB_TOKEN: $(azuresdk-github-pat) + +- script: | + python 'eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py' --db_name "openapiPortal" --collection_name "repoStructureCache" --repos_url_list "https://github.com/Azure/azure-rest-api-specs/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/Azure/azure-rest-api-specs-pr/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSMaster" --repo_clone_path $(System.DefaultWorkingDirectory) + displayName: Update PPE cache + env: + MONGO_CONNECTION_STRING: $(Open-API-MongoDB-Connection-String-PPE) GITHUB_TOKEN: $(azuresdk-github-pat) \ No newline at end of file diff --git a/eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py b/eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py index 39c2e60bccf..d830e88c97c 100644 --- a/eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py +++ b/eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py @@ -66,13 +66,13 @@ def update_mongo_collection( print(f"Using database '{DB_NAME}' and collection '{COLLECTION_NAME}'") try: - existing_document: bool = collection.count_documents({"key": new_document["key"]}) == 0 + existing_document_count: bool = collection.count_documents({"key": new_document["key"]}) except Exception as e: print(f"Error while checking for existing document: {e}") raise e - print(f"Existing document: {existing_document}") - if existing_document: + print(f"Existing documents: {existing_document_count}") + if existing_document_count > 0: print(f"Document for '{new_document['key']}' exists, replacing...") try: collection.replace_one({"key": new_document['key']}, new_document)