[Google Threat Intelligence] Add curated commands and improve polling commands

.circleci/linters_runner.sh → .circleci/analyze_non_packs_files.sh

@@ -13,7 +13,7 @@ echo -e "Folders to be used for lint scan (used by pylint and mypy):\n${all_dirs
 python3 -m ruff $all_1_depth_dirs --select=E,F,PLC,PLE --ignore=PLC1901 || errors=$?
 
 
-echo 'Linter exit code:' $errors
+echo 'analyze non-packs files exit code:' $errors
 if [[ $errors -ne 0 ]]; then
   exit 1
 fi

.circleci/config.yml

@@ -224,36 +224,10 @@ references:
         # poll for neo4j status until available
         while ! curl --fail http://127.0.0.1:7474 &> /dev/null; do sleep 1; done
 
-        ./.circleci/linters_runner.sh
+        ./.circleci/analyze_non_packs_files.sh
         ./.circleci/validate.sh
 
-  run_unit_testing_and_lint: &run_unit_testing_and_lint
-    run:
-      parameters:
-        dockerimageflag:
-          type: string
-      name: Run Unit Testing And Lint - Docker Image:<< parameters.dockerimageflag >>
-      when: always
-      no_output_timeout: 5h
-      command: |
-        if [[ "$(echo "$GCS_MARKET_BUCKET" | tr '[:upper:]' '[:lower:]')" != "marketplace-dist" ]]; then
-          echo "Skipping validations when uploading to a test bucket."
-          exit 0
-        fi
-
-        echo "demisto-sdk version: $(demisto-sdk --version)"
-        echo "mypy version: $(mypy --version)"
-        echo "flake8 py3 version: $(python3 -m flake8 --version)"
-        echo "bandit py3 version: $(python3 -m bandit --version 2>&1)"
-        echo "vulture py3 version: $(python3 -m vulture --version 2>&1)"
-        mkdir ./unit-tests
-
-        neo4j start
-        # poll for neo4j status until available
-        while ! curl --fail http://127.0.0.1:7474 &> /dev/null; do sleep 1; done
-
-        demisto-sdk lint -p 8 -g --test-xml ./unit-tests --log-path ./artifacts --failure-report ./artifacts --coverage-report $ARTIFACTS_FOLDER/coverage_report --docker-image << parameters.dockerimageflag >> --check-dependent-api-module
-
+
   # generate_coverage_reports: &generate_coverage_reports
   #   run:
   #     name: Generate coverage reports
@@ -298,14 +272,6 @@ references:
   nightly_jobs: &nightly_jobs
     - Setup Environment:
         context: nightly_env
-    - Run Unit Testing And Lint:
-        context: nightly_env
-        requires:
-          - Setup Environment
-        matrix:
-          parameters:
-            dockerimageflag: [ "native:ga", "native:maintenance", "native:dev", "from-yml" ]
-        name: Run Unit Testing And Lint - Docker Image:<< matrix.dockerimageflag >>
     - Run Validations:
         requires:
           - Setup Environment
@@ -324,25 +290,6 @@ jobs:
       - *get_contribution_pack
       - *persist_to_workspace
 
-  Run Unit Testing And Lint:
-    <<: *container_config
-    resource_class: large
-    <<: *environment
-    parameters:
-      dockerimageflag:
-        type: string
-    steps:
-      - *attach_workspace
-      - *remote_docker
-      - *install_build_dependencies
-      - *install_node_ci
-      - *install_neo4j
-      - *prepare_environment
-      - *run_unit_testing_and_lint
-      # - *generate_coverage_reports
-      - store_test_results:
-          path: ./unit-tests
-      - *store_artifacts
 
   Run Validations:
     <<: *container_config

.circleci/git_pull_master_into_fork.sh

@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
-
+# this file has been deprecated and relocated to the contribution/utils directory
 #Be aware, only contributors should run this script.
 
+echo "This file has been deprecated and relocated to the contribution/utils directory"
+
 CONTENT_URL='https://github.com/demisto/content.git'
 
 if [ -z "$1" ]

.circleci/is_file_up_to_date.sh

@@ -31,7 +31,7 @@ if [[ $(git diff origin/master -G"." -- ${FILE_TO_CHECK}) ]]; then
         fi
 
         if [[ $BRANCH =~ pull/[0-9]+ ]]; then
-          echo "Run ./.circleci/git_pull_master_into_fork.sh or merge manually from upstream demisto content"
+          echo "Run ./contribution/utils/git_pull_master_into_fork.sh or merge manually from upstream demisto content"
         fi
 
         exit 1

.circleci/update_contribution_pack_in_base_branch.py

@@ -25,7 +25,7 @@ def main():
     github_token = args.github_token
 
     print(
-        f"args received in Utils/update_contribution_pack_in_base_branch.py script: {pr_number=}, {username=}, {repo=}, {branch=}"
+        f"args received in .circleci/update_contribution_pack_in_base_branch.py script: {pr_number=}, {username=}, {repo=}, {branch=}"
     )
 
     packs_dir_names = get_files_from_github(

.github/CODEOWNERS

@@ -9,7 +9,7 @@
 /Tests/Marketplace/corepacks_override.json @yaakovpraisler
 
 # Docker native image
-/Tests/docker_native_image_config.json @JudahSchwartz @samuelFain
+/Tests/docker_native_image_config.json @JudahSchwartz @shmuel44
 
 # Marketplace & Upload-Flow
 /Tests/scripts/create_artifacts_graph/create_artifacts.py @RosenbergYehuda

.github/content_roles.json

@@ -1,14 +1,14 @@
 {
     "CONTRIBUTION_REVIEWERS": [
-        "RotemAmit",
-        "jlevypaloalto",
-        "Shellyber"
+        "barryyosi-panw",
+        "samuelFain",
+        "israelpoli"
     ],
-    "CONTRIBUTION_TL": "BEAdi",
+    "CONTRIBUTION_TL": "thefrieddan1",
     "CONTRIBUTION_SECURITY_REVIEWER": "ssokolovich",
     "ON_CALL_DEVS": [
-        "acarmi",
-        "ypreisler"
+        "skidorball",
+        "ayousef"
     ],
     "DOC_REVIEWER": "ShirleyDenkberg",
     "TIM_REVIEWER": "MLainer1"

Utils/github_workflow_scripts/autobump_release_notes/autobump_rn.py → .github/github_workflow_scripts/autobump_rn.py

@@ -8,11 +8,11 @@
 from github.PullRequest import PullRequest
 from github.Repository import Repository
 import sys
-from Utils.github_workflow_scripts.autobump_release_notes.skip_conditions import MetadataCondition, \
+from skip_conditions import MetadataCondition, \
     LastModifiedCondition, LabelCondition, AddedRNFilesCondition, HasConflictOnAllowedFilesCondition, \
     PackSupportCondition, MajorChangeCondition, MaxVersionCondition, OnlyVersionChangedCondition, \
     OnlyOneRNPerPackCondition, SameRNMetadataVersionCondition, AllowedBumpCondition, UpdateType
-from Utils.github_workflow_scripts.utils import timestamped_print, Checkout
+from utils import timestamped_print, Checkout
 from git import Repo
 from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN
 import os
@@ -192,6 +192,10 @@ def manage(self):
         for pr in self.github_repo_obj.get_pulls(
             state="open", sort="created", base=BASE
         ):
+            if pr.draft:
+                # The bot does not go through a PR that is in draft
+                continue
+
             print(
                 f"{t.yellow}Looking on pr number [{pr.number}]: last updated: "
                 f"{str(pr.updated_at)}, branch={pr.head.ref}"

Utils/github_workflow_scripts/check_if_partner_approved_label_exists.py → .github/github_workflow_scripts/check_if_partner_approved_label_exists.py

@@ -7,7 +7,7 @@
 from github.Repository import Repository
 from github.PullRequest import PullRequest
 from demisto_sdk.commands.common.tools import get_pack_name
-from Utils.github_workflow_scripts.utils import timestamped_print, get_support_level
+from utils import timestamped_print, get_support_level
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 print = timestamped_print

Utils/github_workflow_scripts/create_internal_pr.py → .github/github_workflow_scripts/create_internal_pr.py

@@ -7,12 +7,11 @@
 from github import Github
 from handle_external_pr import EXTERNAL_LABEL
 
-from Utils.github_workflow_scripts.utils import (
+from utils import (
     get_env_var,
     timestamped_print,
-    load_json,
     get_doc_reviewer,
-    CONTENT_ROLES_PATH
+    get_content_roles
 )
 from urllib3.exceptions import InsecureRequestWarning
 
@@ -81,7 +80,7 @@ def main():
     assignees = [assignee.login for assignee in merged_pr.assignees]
 
     # Un-assign the tech writer (cause the docs reviewed has already been done on the external PR)
-    content_roles = load_json(CONTENT_ROLES_PATH)
+    content_roles = get_content_roles()
     if content_roles:
 
         try:
@@ -96,7 +95,7 @@ def main():
             print(f"{str(ve)}. Skipped tech writer unassignment.")
 
     else:
-        print(f"Unable to parse JSON from '{CONTENT_ROLES_PATH}'. Skipping tech writer unassignment.")
+        print("Unable to get content roles. Skipping tech writer unassignment...")
 
     pr.add_to_assignees(*assignees)
     print(f'{t.cyan}Assigned users {assignees}{t.normal}')

Utils/github_workflow_scripts/github_workflow_scripts_tests/autobump_rn_test.py → .github/github_workflow_scripts/github_workflow_scripts_tests/autobump_rn_test.py

@@ -5,17 +5,17 @@
 from typing import Optional, List
 from unittest.mock import MagicMock
 import pytest
-from Utils.github_workflow_scripts.autobump_release_notes.autobump_rn import (
+from github_workflow_scripts.autobump_release_notes.autobump_rn import (
     PackAutoBumper,
     BranchAutoBumper, AutoBumperManager,
 )
-from Utils.github_workflow_scripts.autobump_release_notes.skip_conditions import ConditionResult, MetadataCondition, \
+from github_workflow_scripts.autobump_release_notes.skip_conditions import ConditionResult, MetadataCondition, \
     LastModifiedCondition, LabelCondition, AddedRNFilesCondition, HasConflictOnAllowedFilesCondition, \
     PackSupportCondition, MajorChangeCondition, MaxVersionCondition, OnlyVersionChangedCondition, \
     OnlyOneRNPerPackCondition, SameRNMetadataVersionCondition, AllowedBumpCondition, UpdateType
 from git import GitCommandError
 from demisto_sdk.commands.update_release_notes.update_rn import UpdateRN
-from Utils.github_workflow_scripts.autobump_release_notes import skip_conditions
+from github_workflow_scripts.autobump_release_notes import skip_conditions
 MERGE_STDOUT = "stdout: '\n Auto-merging {}\n failed.\n Auto-merging {}\n failed.\n"
 
 
@@ -163,7 +163,7 @@ def test_get_metadata_files(mocker):
     origin_metadata = {"name": "MyPack", "currentVersion": "1.0.5"}
     branch_metadata = {"name": "MyPack", "currentVersion": "1.0.4"}
     base_metadata = {"name": "MyPack", "currentVersion": "1.0.3"}
-    mocker.patch("Utils.github_workflow_scripts.autobump_release_notes.skip_conditions.Checkout")
+    mocker.patch("github_workflow_scripts.autobump_release_notes.skip_conditions.Checkout")
     mocker.patch.object(
         skip_conditions,
         "load_json",
@@ -614,7 +614,7 @@ def test_branch_auto_bumper(mocker):
     pack_auto_bumper = MagicMock()
     pack_auto_bumper.autobump.return_value = "1.0.2"
     pack_auto_bumper.pack_id = "MyPack"
-    mocker.patch("Utils.github_workflow_scripts.autobump_release_notes.autobump_rn.Checkout")
+    mocker.patch("github_workflow_scripts.autobump_release_notes.autobump_rn.Checkout")
     branch_auto_bumper = BranchAutoBumper(
         pr=PullRequest(),
         git_repo=Repo(),
@@ -647,7 +647,7 @@ def test_autobump_manager(mocker):
         git_repo_obj=Repo(files=CHANGED_FILES),
         run_id="1",
     )
-    mocker.patch("Utils.github_workflow_scripts.autobump_release_notes.autobump_rn.Checkout")
+    mocker.patch("github_workflow_scripts.autobump_release_notes.autobump_rn.Checkout")
     mocker.patch.object(BranchAutoBumper, "autobump")
     mocker.patch.object(
         MetadataCondition,

Utils/github_workflow_scripts/github_workflow_scripts_tests/check_if_partner_approved_label_test.py → .github/github_workflow_scripts/github_workflow_scripts_tests/check_if_partner_approved_label_test.py

@@ -4,11 +4,11 @@
 @pytest.mark.parametrize(
     'support_levels, expected_support_level', [
         (
-            ['Utils/github_workflow_scripts/github_workflow_scripts_tests/test_files/Packs/Pack1/pack_metadata.json'],
+            ['.github/github_workflow_scripts/github_workflow_scripts_tests/test_files/Packs/Pack1/pack_metadata.json'],
             {'xsoar'}
         ),
         (
-            ['Utils/github_workflow_scripts/github_workflow_scripts_tests/test_files/Packs/Pack2/pack_metadata.json'],
+            ['.github/github_workflow_scripts/github_workflow_scripts_tests/test_files/Packs/Pack2/pack_metadata.json'],
             {'partner'}
         ),
     ]
@@ -24,5 +24,5 @@ def test_get_support_level(support_levels, expected_support_level):
         Then:
             - make sure the highest support level is always returned
     """
-    from Utils.github_workflow_scripts.check_if_partner_approved_label_exists import get_support_level
+    from github_workflow_scripts.check_if_partner_approved_label_exists import get_support_level
     assert get_support_level(support_levels) == expected_support_level

Utils/github_workflow_scripts/github_workflow_scripts_tests/handle_external_pr_test.py → .github/github_workflow_scripts/github_workflow_scripts_tests/handle_external_pr_test.py

@@ -1,7 +1,7 @@
 import os
 import pytest
 from typing import Final
-from Utils.github_workflow_scripts.handle_external_pr import is_requires_security_reviewer, get_location_of_reviewer
+from github_workflow_scripts.handle_external_pr import is_requires_security_reviewer, get_location_of_reviewer
 
 
 INTEGRATION_PATH: Final[str] = 'Packs/HelloWorld/Integrations/HelloWorld/HelloWorld.py'
@@ -68,7 +68,7 @@ def test_get_highest_support_label(support_levels, expected_support_level):
     Then:
         - make sure the highest support level is always returned
     """
-    from Utils.github_workflow_scripts.handle_external_pr import get_highest_support_label
+    from github_workflow_scripts.handle_external_pr import get_highest_support_label
     assert get_highest_support_label(support_levels) == expected_support_level
 
 
@@ -90,15 +90,15 @@ def test_get_packs_support_level_label(mocker, fork_owner, expected_fork_owner):
         - make sure correct support label is returned.
         - fork owner that is being delivered to the Checkout branch is correct.
     """
-    from Utils.github_workflow_scripts.handle_external_pr import get_packs_support_level_label, Checkout
-    from Utils.github_workflow_scripts.utils import ChangeCWD
+    from github_workflow_scripts.handle_external_pr import get_packs_support_level_label, Checkout
+    from github_workflow_scripts.utils import ChangeCWD
 
     checkout_mocker = mocker.patch.object(Checkout, '__init__', return_value=None)
     mocker.patch.object(Checkout, '__enter__', return_value=None)
     mocker.patch.object(Checkout, '__exit__', return_value=None)
     mocker.patch.object(os, 'getenv', return_value=fork_owner)
 
-    with ChangeCWD('Utils/github_workflow_scripts/github_workflow_scripts_tests/test_files'):
+    with ChangeCWD('.github/github_workflow_scripts/github_workflow_scripts_tests/test_files'):
         assert get_packs_support_level_label(
             file_paths=['Packs/Pack1/pack_metadata.json'], external_pr_branch='test'
         ) == 'Xsoar Support Level'
@@ -117,12 +117,12 @@ def test_get_packs_support_level_label_checkout_failed(mocker):
     Then:
         - make sure correct support label is still returned.
     """
-    from Utils.github_workflow_scripts.handle_external_pr import get_packs_support_level_label, Checkout
-    from Utils.github_workflow_scripts.utils import ChangeCWD
+    from github_workflow_scripts.handle_external_pr import get_packs_support_level_label, Checkout
+    from github_workflow_scripts.utils import ChangeCWD
 
     mocker.patch.object(Checkout, '__init__', return_value=Exception('Error'))
 
-    with ChangeCWD('Utils/github_workflow_scripts/github_workflow_scripts_tests/test_files'):
+    with ChangeCWD('.github/github_workflow_scripts/github_workflow_scripts_tests/test_files'):
         assert get_packs_support_level_label(
             file_paths=['Packs/Pack1/pack_metadata.json'], external_pr_branch='test'
         ) == 'Xsoar Support Level'

Utils/github_workflow_scripts/github_workflow_scripts_tests/link_pr_to_jira_issue_test.py → .github/github_workflow_scripts/github_workflow_scripts_tests/link_pr_to_jira_issue_test.py

@@ -1,6 +1,6 @@
 import pytest
 
-from Utils.github_workflow_scripts.jira_integration_scripts import link_pr_to_jira_issue
+from github_workflow_scripts.jira_integration_scripts import link_pr_to_jira_issue
 
 PR_WITH_ONLY_FIXES_WITH_SPACE = """This pr is dummy
 fixes: https://jira-hq.paloaltonetworks.local/browse/CIAC-3473 somthing