DCA-2372 jira api users search

app/util/api/jira_clients.py

@@ -1,10 +1,11 @@
+import json
+import string
 from selenium.common.exceptions import WebDriverException
 
 from util.api.abstract_clients import RestClient, JSM_EXPERIMENTAL_HEADERS
 from selenium_ui.conftest import retry
 
 BATCH_SIZE_BOARDS = 1000
-BATCH_SIZE_USERS = 1000
 BATCH_SIZE_ISSUES = 1000
 
 
@@ -48,38 +49,53 @@ def get_boards(self, start_at=0, max_results=100, board_type=None, name=None, pr
         return boards_list
 
     @retry()
-    def get_users(self, username='.', start_at=0, max_results=1000, include_active=True, include_inactive=False):
+    def get_users(self, username='.', include_active=True, include_inactive=False):
         """
-        Returns a list of users that match the search string. This resource cannot be accessed anonymously.
-        :param username: A query string used to search username, name or e-mail address. "." - search for all users.
-        :param start_at: the index of the first user to return (0-based).
-        :param max_results: the maximum number of users to return (defaults to 50).
-        The maximum allowed value is 1000.
-        If you specify a value that is higher than this number, your search results will be truncated.
-        :param include_active: If true, then active users are included in the results (default true)
-        :param include_inactive: If true, then inactive users are included in the results (default false)
-        :return: Returns the requested users
+        Starting from Jira 10 there is no way to get more than 100 users with get_users() API
+        startAt param will be deprecated in Jira 10.3+
         """
+        max_results = 100
 
-        loop_count = max_results // BATCH_SIZE_USERS + 1
-        last_loop_remainder = max_results % BATCH_SIZE_USERS
-
-        users_list = list()
-        max_results = BATCH_SIZE_USERS if max_results > BATCH_SIZE_USERS else max_results
-
-        while loop_count > 0:
-            api_url = f'{self.host}/rest/api/2/user/search?username={username}&startAt={start_at}' \
-                      f'&maxResults={max_results}&includeActive={include_active}&includeInactive={include_inactive}'
-            response = self.get(api_url, "Could not retrieve users")
-
-            users_list.extend(response.json())
-            loop_count -= 1
-            start_at += len(response.json())
-            if loop_count == 1:
-                max_results = last_loop_remainder
+        api_url = f'{self.host}/rest/api/2/user/search?username={username}' \
+                  f'&maxResults={max_results}' \
+                  f'&includeActive={include_active}' \
+                  f'&includeInactive={include_inactive}'
+        response = self.get(api_url, "Could not retrieve users")
+        users_list = response.json()
 
         return users_list
 
+    @retry()
+    def get_users_by_name_search(self, username, users_count, include_active=True, include_inactive=False):
+        """
+        Starting from Jira 10 there is no way to get more than 100 users with get_users() API
+        Getting more than 100 users by batch search.
+        """
+        print(f"INFO: Users search. Prefix: '{username}', users_count: {users_count}")
+        perf_users = list()
+
+        first_100 = self.get_users(username=username, include_active=True, include_inactive=False)
+        if users_count <= 100 or len(first_100) < 100:
+            perf_users = first_100[:users_count]
+        else:
+            name_start_list = list(string.digits + "_" + string.ascii_lowercase)
+            for i in name_start_list:
+                users_batch = self.get_users(username=username+i, include_active=True, include_inactive=False)
+                if len(users_batch) == 100:
+                    print(f"Warning: found 100 users starts with: {username+i}. Checking if there are more.")
+                    users_batch = self.get_users_by_name_search(username=username+i,
+                                                                users_count=users_count-len(perf_users))
+                perf_users.extend(users_batch)
+
+                # get rid of any duplicates by creating a set from json objects
+                set_of_jsons = {json.dumps(d, sort_keys=True) for d in perf_users}
+                perf_users = [json.loads(t) for t in set_of_jsons]
+                print(f"INFO: Current found users count: {len(perf_users)}")
+
+                if len(perf_users) >= users_count:
+                    perf_users = perf_users[:users_count]
+                    break
+        return perf_users
 
     @retry()
     def issues_search(self, jql='order by key', start_at=0, max_results=1000, fields=None):

app/util/data_preparation/jira_prepare_data.py

@@ -137,7 +137,8 @@ def __get_boards(jira_api, board_type):
 
 
 def __get_users(jira_api):
-    perf_users = jira_api.get_users(username=DEFAULT_USER_PREFIX, max_results=performance_users_count)
+    perf_users = jira_api.get_users_by_name_search(username=DEFAULT_USER_PREFIX, users_count=performance_users_count)
+
     users = generate_perf_users(api=jira_api, cur_perf_user=perf_users)
     if not users:
         raise SystemExit(f"There are no users in Jira accessible by a random performance user: {jira_api.user}")

app/util/data_preparation/jsm_prepare_data.py

@@ -17,7 +17,7 @@
 MAX_WORKERS = None
 
 ERROR_LIMIT = 10
-DEFAULT_AGENT_PREFIX = 'performance_agent_'
+DEFAULT_AGENT_PREFIX = 'performance_agent_0_'
 DEFAULT_AGENT_APP_KEYS = ["jira-servicedesk"]
 DEFAULT_CUSTOMER_PREFIX = 'performance_customer_'
 DEFAULT_PASSWORD = 'password'
@@ -93,15 +93,16 @@ def __filter_customer_with_requests(customer, jsm_client):
 def __get_customers_with_requests(jira_client, jsm_client, count):
     customers_with_requests = []
     customers_without_requests = []
-    start_at = 0
-    max_count_iteration = 1000
+    count_to_search = 0
+    limit_users_to_search = 3000
+    max_count_iteration = 500
     customers_chunk_size = 150
-    while len(customers_with_requests) < count:
-        customers = jira_client.get_users(username=DEFAULT_CUSTOMER_PREFIX, max_results=max_count_iteration,
-                                          start_at=start_at)
+    while len(customers_with_requests) < count and count_to_search < limit_users_to_search:
+        count_to_search += max_count_iteration
+        customers = jira_client.get_users_by_name_search(username=DEFAULT_CUSTOMER_PREFIX,
+                                                         users_count=count_to_search)
         if not customers:
             break
-        start_at = start_at + max_count_iteration
         customer_chunks = [customers[x:x + customers_chunk_size]
                            for x in range(0, len(customers), customers_chunk_size)]
 
@@ -117,26 +118,34 @@ def __get_customers_with_requests(jira_client, jsm_client, count):
                 if customer_data['has_requests']:
                     if len(customers_with_requests) >= count:
                         break
-                    customers_with_requests.append(customer_data)
+                    if customer_data['name'] not in [i['name'] for i in customers_with_requests]:
+                        customers_with_requests.append(customer_data)
                 else:
                     customers_without_requests.append(customer_data)
 
-    print(f'Retrieved customers with requests: {len(customers_with_requests)}')
+    customers_with_requests_count = len(customers_with_requests)
+    print(f'Retrieved customers with requests: {customers_with_requests_count}')
+
+    if customers_with_requests_count < count:
+        raise Exception(f"ERROR: Found {customers_with_requests_count} customers with open requests, "
+                        f"but 'concurrency_customers' in jsm.yml is {count}. "
+                        f"Create more customers with open requests or reduce 'concurrency_customers'.")
 
     return customers_with_requests
 
 
 @print_timing('Retrieving agents')
 def __get_agents(jira_client):
-    prefix_name, application_keys, count = DEFAULT_AGENT_PREFIX, DEFAULT_AGENT_APP_KEYS, performance_agents_count
-    perf_users = jira_client.get_users(username=prefix_name, max_results=count)
-    users_to_create = count - len(perf_users)
+    perf_users = jira_client.get_users_by_name_search(username=DEFAULT_AGENT_PREFIX,
+                                                      users_count=performance_agents_count)
+
+    users_to_create = performance_agents_count - len(perf_users)
     if users_to_create > 0:
-        add_users = __generate_users(api=jira_client, num_to_create=users_to_create, prefix_name=prefix_name,
-                                     application_keys=application_keys)
+        add_users = __generate_users(api=jira_client, num_to_create=users_to_create, prefix_name=DEFAULT_AGENT_PREFIX,
+                                     application_keys=DEFAULT_AGENT_APP_KEYS)
         if not add_users:
             raise Exception(f"ERROR: Jira Service Management could not create agent"
-                            f"There were {len(perf_users)}/{count} retrieved.")
+                            f"There were {len(perf_users)}/{performance_agents_count} retrieved.")
         perf_users.extend(add_users)
     return perf_users
 
@@ -145,14 +154,14 @@ def __get_agents(jira_client):
 def __get_customers(jira_client, jsm_client, servicedesks):
     created_agents = []
     errors_count = 0
-    prefix_name, application_keys, count = DEFAULT_CUSTOMER_PREFIX, None, performance_customers_count
     perf_users_with_requests = __get_customers_with_requests(jsm_client=jsm_client, jira_client=jira_client,
-                                                             count=count)
+                                                             count=performance_customers_count)
+
     while len(perf_users_with_requests) < performance_customers_count:
-        username = f"{prefix_name}{__generate_random_string(10)}"
+        username = f"{DEFAULT_CUSTOMER_PREFIX}{__generate_random_string(10)}"
         try:
             agent = jira_client.create_user(name=username, password=DEFAULT_PASSWORD,
-                                            application_keys=application_keys)
+                                            application_keys=None)
             created_agents.append(agent)
             request_types = __get_request_types(jsm_client, servicedesks)
             if not request_types: