From 691e715ac4d0294e4d5dc3a4603968c8d220e97b Mon Sep 17 00:00:00 2001 From: Jonathan Cardenas Date: Thu, 22 Feb 2024 11:28:36 -0800 Subject: [PATCH 1/5] Update repo URLs and requirements file path for cache job --- eng/pipelines/openapitools/repo-structure-cache-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/openapitools/repo-structure-cache-job.yml b/eng/pipelines/openapitools/repo-structure-cache-job.yml index 392b72690c2..295d02dce49 100644 --- a/eng/pipelines/openapitools/repo-structure-cache-job.yml +++ b/eng/pipelines/openapitools/repo-structure-cache-job.yml @@ -9,7 +9,7 @@ parameters: - name: REPOS_URL_LIST type: string - default: "https://github.com/test-repo-billy/azure-rest-api-specs/tree/main,https://github.com/test-repo-billy/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/test-repo-billy/azure-rest-api-specs-pr/tree/main,https://github.com/test-repo-billy/azure-rest-api-specs/tree/RPSaaSDev" + default: "https://github.com/azure/azure-rest-api-specs/tree/main,https://github.com/azure/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/azure/azure-rest-api-specs-pr/tree/main,https://github.com/azure/azure-rest-api-specs/tree/RPSaaSDev" trigger: - main @@ -21,7 +21,7 @@ pool: steps: - script: | - pip install -r 'eng/scripts/OpenAPI Hub/requirements.txt' + pip install -r 'eng/scripts/openapitools/python/cacherepostructure/requirements.txt' pip list python 'eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py' --db_name ${{ parameters.DB_NAME }} --collection_name ${{ parameters.COLLECTION_NAME }} --repos_url_list ${{ parameters.REPOS_URL_LIST }} --repo_clone_path $(System.DefaultWorkingDirectory) displayName: Run Script From 09d2dbe081b049a562ed0370281a7e1fbbd780eb Mon Sep 17 00:00:00 2001 From: Jonathan Cardenas Date: Thu, 22 Feb 2024 15:07:48 -0800 Subject: [PATCH 2/5] Fixing bug in script --- .../python/cacherepostructure/sync-repo-structure-cache.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py b/eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py index 39c2e60bccf..d830e88c97c 100644 --- a/eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py +++ b/eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py @@ -66,13 +66,13 @@ def update_mongo_collection( print(f"Using database '{DB_NAME}' and collection '{COLLECTION_NAME}'") try: - existing_document: bool = collection.count_documents({"key": new_document["key"]}) == 0 + existing_document_count: bool = collection.count_documents({"key": new_document["key"]}) except Exception as e: print(f"Error while checking for existing document: {e}") raise e - print(f"Existing document: {existing_document}") - if existing_document: + print(f"Existing documents: {existing_document_count}") + if existing_document_count > 0: print(f"Document for '{new_document['key']}' exists, replacing...") try: collection.replace_one({"key": new_document['key']}, new_document) From c8f4a0814d23c00c8b99c7adc4cc41b472539bde Mon Sep 17 00:00:00 2001 From: Jonathan Cardenas Date: Thu, 22 Feb 2024 15:08:24 -0800 Subject: [PATCH 3/5] Update yaml to match PPE and PROD database names and repo branches --- eng/pipelines/openapitools/repo-structure-cache-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/openapitools/repo-structure-cache-job.yml b/eng/pipelines/openapitools/repo-structure-cache-job.yml index 295d02dce49..c0c88bdeace 100644 --- a/eng/pipelines/openapitools/repo-structure-cache-job.yml +++ b/eng/pipelines/openapitools/repo-structure-cache-job.yml @@ -1,7 +1,7 @@ parameters: - name: DB_NAME type: string - default: "openapi-portal" + default: "openapiPortal" - name: COLLECTION_NAME type: string @@ -9,7 +9,7 @@ parameters: - name: REPOS_URL_LIST type: string - default: "https://github.com/azure/azure-rest-api-specs/tree/main,https://github.com/azure/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/azure/azure-rest-api-specs-pr/tree/main,https://github.com/azure/azure-rest-api-specs/tree/RPSaaSDev" + default: "https://github.com/azure/azure-rest-api-specs/tree/main,https://github.com/azure/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/azure/azure-rest-api-specs-pr/tree/main,https://github.com/azure/azure-rest-api-specs-pr/tree/RPSaaSMaster" trigger: - main From 2b12fb1a96f780396d58003663f74ffcb565c701 Mon Sep 17 00:00:00 2001 From: Jonathan Cardenas Date: Thu, 22 Feb 2024 15:10:46 -0800 Subject: [PATCH 4/5] Use git owner uppercase to match DB key in PPE and PROD --- eng/pipelines/openapitools/repo-structure-cache-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/openapitools/repo-structure-cache-job.yml b/eng/pipelines/openapitools/repo-structure-cache-job.yml index c0c88bdeace..e6c3987bdb9 100644 --- a/eng/pipelines/openapitools/repo-structure-cache-job.yml +++ b/eng/pipelines/openapitools/repo-structure-cache-job.yml @@ -9,7 +9,7 @@ parameters: - name: REPOS_URL_LIST type: string - default: "https://github.com/azure/azure-rest-api-specs/tree/main,https://github.com/azure/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/azure/azure-rest-api-specs-pr/tree/main,https://github.com/azure/azure-rest-api-specs-pr/tree/RPSaaSMaster" + default: "https://github.com/Azure/azure-rest-api-specs/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/Azure/azure-rest-api-specs-pr/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSMaster" trigger: - main From 08a3ceabc2a213c1285aecded8543d8422b9ac54 Mon Sep 17 00:00:00 2001 From: Jonathan Cardenas Date: Thu, 22 Feb 2024 16:05:21 -0800 Subject: [PATCH 5/5] Update both PPE and PROD in different steps using the same pipeline --- .../openapitools/repo-structure-cache-job.yml | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/eng/pipelines/openapitools/repo-structure-cache-job.yml b/eng/pipelines/openapitools/repo-structure-cache-job.yml index e6c3987bdb9..debcb2075da 100644 --- a/eng/pipelines/openapitools/repo-structure-cache-job.yml +++ b/eng/pipelines/openapitools/repo-structure-cache-job.yml @@ -1,30 +1,26 @@ -parameters: -- name: DB_NAME - type: string - default: "openapiPortal" - -- name: COLLECTION_NAME - type: string - default: "repoStructureCache" - -- name: REPOS_URL_LIST - type: string - default: "https://github.com/Azure/azure-rest-api-specs/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/Azure/azure-rest-api-specs-pr/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSMaster" - -trigger: -- main +trigger: none +pr: none pool: name: azsdk-pool-mms-ubuntu-2204-general vmImage: ubuntu-22.04 steps: - - script: | pip install -r 'eng/scripts/openapitools/python/cacherepostructure/requirements.txt' pip list - python 'eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py' --db_name ${{ parameters.DB_NAME }} --collection_name ${{ parameters.COLLECTION_NAME }} --repos_url_list ${{ parameters.REPOS_URL_LIST }} --repo_clone_path $(System.DefaultWorkingDirectory) - displayName: Run Script + displayName: Install dependencies + +- script: | + python 'eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py' --db_name "openapiPortal" --collection_name "repoStructureCache" --repos_url_list "https://github.com/Azure/azure-rest-api-specs/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/Azure/azure-rest-api-specs-pr/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSMaster" --repo_clone_path $(System.DefaultWorkingDirectory) + displayName: Update PROD cache env: MONGO_CONNECTION_STRING: $(Open-API-MongoDB-Connection-String) + GITHUB_TOKEN: $(azuresdk-github-pat) + +- script: | + python 'eng/scripts/openapitools/python/cacherepostructure/sync-repo-structure-cache.py' --db_name "openapiPortal" --collection_name "repoStructureCache" --repos_url_list "https://github.com/Azure/azure-rest-api-specs/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSDev,https://github.com/Azure/azure-rest-api-specs-pr/tree/main,https://github.com/Azure/azure-rest-api-specs-pr/tree/RPSaaSMaster" --repo_clone_path $(System.DefaultWorkingDirectory) + displayName: Update PPE cache + env: + MONGO_CONNECTION_STRING: $(Open-API-MongoDB-Connection-String-PPE) GITHUB_TOKEN: $(azuresdk-github-pat) \ No newline at end of file