Fix make lint on Python 3.10 (#1565)

* Fix `make lint` on Python 3.10

In practice this means upgrading pylint which was failing on 3.9 and
3.10.

* Disable duplicate-code pylint check

.pylintrc

@@ -46,11 +46,6 @@ suggestion-mode=yes
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no
 
-# pylint-quotes
-string-quote=double
-triple-quote=double
-docstring-quote=double
-
 [MESSAGES CONTROL]
 
 # Only show warnings with the listed confidence levels. Leave empty to show
@@ -66,85 +61,14 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use "--disable=all --enable=classes
 # --disable=W".
-disable=print-statement,
-        parameter-unpacking,
-        unpacking-in-except,
-        old-raise-syntax,
-        backtick,
-        long-suffix,
-        old-ne-operator,
-        old-octal-literal,
-        import-star-module-level,
-        non-ascii-bytes-literal,
-        raw-checker-failed,
+disable=raw-checker-failed,
         bad-inline-option,
         locally-disabled,
         file-ignored,
         suppressed-message,
         useless-suppression,
         deprecated-pragma,
         use-symbolic-message-instead,
-        apply-builtin,
-        basestring-builtin,
-        buffer-builtin,
-        cmp-builtin,
-        coerce-builtin,
-        execfile-builtin,
-        file-builtin,
-        long-builtin,
-        raw_input-builtin,
-        reduce-builtin,
-        standarderror-builtin,
-        unicode-builtin,
-        xrange-builtin,
-        coerce-method,
-        delslice-method,
-        getslice-method,
-        setslice-method,
-        no-absolute-import,
-        old-division,
-        dict-iter-method,
-        dict-view-method,
-        next-method-called,
-        metaclass-assignment,
-        indexing-exception,
-        raising-string,
-        reload-builtin,
-        oct-method,
-        hex-method,
-        nonzero-method,
-        cmp-method,
-        input-builtin,
-        round-builtin,
-        intern-builtin,
-        unichr-builtin,
-        map-builtin-not-iterating,
-        zip-builtin-not-iterating,
-        range-builtin-not-iterating,
-        filter-builtin-not-iterating,
-        using-cmp-argument,
-        eq-without-hash,
-        div-method,
-        idiv-method,
-        rdiv-method,
-        exception-message-attribute,
-        invalid-str-codec,
-        sys-max-int,
-        bad-python3-import,
-        deprecated-string-function,
-        deprecated-str-translate-call,
-        deprecated-itertools-function,
-        deprecated-types-field,
-        next-method-defined,
-        dict-items-not-iterating,
-        dict-keys-not-iterating,
-        dict-values-not-iterating,
-        deprecated-operator-function,
-        deprecated-urllib-function,
-        xreadlines-attribute,
-        deprecated-sys-function,
-        exception-escape,
-        comprehension-escape,
         # custom rally ignores start here
         invalid-name,
         missing-module-docstring,
@@ -153,27 +77,28 @@ disable=print-statement,
         too-many-arguments,
         too-many-locals,
         fixme,
-        no-self-use,
         too-many-branches,
         too-few-public-methods,
         redefined-builtin,
         global-statement,
         broad-except,
         no-else-return,
-        no-else-raise,
         no-else-continue,
         no-member,
         too-many-instance-attributes,
         too-many-statements,
         inconsistent-return-statements,
-        C0302,
-        C4001,
-        R0916,
-        W0201,
-        W0613,
-        W0621,
-        invalid-docstring-quote,
-        raise-missing-from
+        raise-missing-from,
+        duplicate-code,
+        unspecified-encoding,
+        consider-using-dict-items,
+        too-many-lines,
+        too-many-boolean-expressions,
+        attribute-defined-outside-init,
+        unused-argument,
+        redefined-outer-name,
+        consider-using-f-string,  # pyupgrade will cover that
+        use-implicit-booleaness-not-comparison,
 
 
 # Enable the message, report, category or checker with the given id(s). You can
@@ -365,13 +290,6 @@ max-line-length=140
 # Maximum number of lines in a module.
 max-module-lines=1000
 
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=trailing-comma,
-               dict-separator
-
 # Allow the body of a class to be on the same line as the declaration if body
 # contains single statement.
 single-line-class-stmt=no

benchmarks/track/bulk_params_test.py

@@ -87,10 +87,8 @@ def create_reader(bulk_size):
     disable_gc=True,
 )
 def test_index_data_reader_100(benchmark):
-    reader = create_reader(bulk_size=100)
-    reader.__enter__()
-    benchmark(reader.__next__)
-    reader.__exit__(None, None, None)
+    with create_reader(bulk_size=100) as reader:
+        benchmark(reader.__next__)
 
 
 @pytest.mark.benchmark(
@@ -100,10 +98,8 @@ def test_index_data_reader_100(benchmark):
     disable_gc=True,
 )
 def test_index_data_reader_1000(benchmark):
-    reader = create_reader(bulk_size=1000)
-    reader.__enter__()
-    benchmark(reader.__next__)
-    reader.__exit__(None, None, None)
+    with create_reader(bulk_size=1000) as reader:
+        benchmark(reader.__next__)
 
 
 @pytest.mark.benchmark(
@@ -113,10 +109,8 @@ def test_index_data_reader_1000(benchmark):
     disable_gc=True,
 )
 def test_index_data_reader_10000(benchmark):
-    reader = create_reader(bulk_size=10000)
-    reader.__enter__()
-    benchmark(reader.__next__)
-    reader.__exit__(None, None, None)
+    with create_reader(bulk_size=10000) as reader:
+        benchmark(reader.__next__)
 
 
 @pytest.mark.benchmark(
@@ -126,7 +120,5 @@ def test_index_data_reader_10000(benchmark):
     disable_gc=True,
 )
 def test_index_data_reader_100000(benchmark):
-    reader = create_reader(bulk_size=100000)
-    reader.__enter__()
-    benchmark(reader.__next__)
-    reader.__exit__(None, None, None)
+    with create_reader(bulk_size=100000) as reader:
+        benchmark(reader.__next__)

esrally/client/factory.py

@@ -126,8 +126,7 @@ def __init__(self, hosts, client_options):
                         "to 'create_api_key_per_client' in order to create client API keys."
                     )
                 )
-            else:
-                self.logger.info("Automatic creation of client API keys: on")
+            self.logger.info("Automatic creation of client API keys: on")
         else:
             self.logger.info("Automatic creation of client API keys: off")
 
@@ -254,9 +253,8 @@ def wait_for_rest_layer(es, max_attempts=40):
         except elasticsearch.ConnectionError as e:
             if "SSL: UNKNOWN_PROTOCOL" in str(e):
                 raise exceptions.SystemSetupError("Could not connect to cluster via https. Is this an https endpoint?", e)
-            else:
-                logger.debug("Got connection error on attempt [%s]. Sleeping...", attempt)
-                time.sleep(3)
+            logger.debug("Got connection error on attempt [%s]. Sleeping...", attempt)
+            time.sleep(3)
         except elasticsearch.TransportError as e:
             # cluster block, x-pack not initialized yet, our wait condition is not reached
             if e.status_code in (503, 401, 408):
@@ -292,9 +290,8 @@ def create_api_key(es, client_id, max_attempts=5):
                 raise exceptions.SystemSetupError(
                     "Got status code 405 when attempting to create API keys. Is Elasticsearch Security enabled?", e
                 )
-            else:
-                logger.debug("Got status code [%s] on attempt [%s] of [%s]. Sleeping...", e.status_code, attempt, max_attempts)
-                time.sleep(1)
+            logger.debug("Got status code [%s] on attempt [%s] of [%s]. Sleeping...", e.status_code, attempt, max_attempts)
+            time.sleep(1)
 
 
 def delete_api_keys(es, ids, max_attempts=5):
@@ -312,8 +309,7 @@ def raise_exception(failed_ids, cause=None):
         msg = f"Could not delete API keys with the following IDs: {failed_ids}"
         if cause is not None:
             raise exceptions.RallyError(msg) from cause
-        else:
-            raise exceptions.RallyError(msg)
+        raise exceptions.RallyError(msg)
 
     # Before ES 7.10, deleting API keys by ID had to be done individually.
     # After ES 7.10, a list of API key IDs can be deleted in one request.

esrally/driver/runner.py

@@ -2274,7 +2274,8 @@ async def __call__(self, es, params):
             if state == "failed":
                 failure_reason = stats_response["transforms"][0].get("reason", "unknown")
                 raise exceptions.RallyAssertionError(f"Transform [{transform_id}] failed with [{failure_reason}].")
-            elif state == "stopped" or wait_for_completion is False:
+
+            if state == "stopped" or wait_for_completion is False:
                 self._completed = True
                 self._percent_completed = 1.0
             else:
@@ -2813,12 +2814,12 @@ async def __call__(self, es, params):
             except (socket.timeout, elasticsearch.exceptions.ConnectionError):
                 if last_attempt or not retry_on_timeout:
                     raise
-                else:
-                    await asyncio.sleep(sleep_time)
+                await asyncio.sleep(sleep_time)
             except elasticsearch.exceptions.TransportError as e:
                 if last_attempt or not retry_on_timeout:
                     raise e
-                elif e.status_code == 408:
+
+                if e.status_code == 408:
                     self.logger.info("[%s] has timed out. Retrying in [%.2f] seconds.", repr(self.delegate), sleep_time)
                     await asyncio.sleep(sleep_time)
                 else:

esrally/mechanic/mechanic.py

@@ -72,7 +72,7 @@ def install(cfg):
     elif build_type == "docker":
         if len(plugins) > 0:
             raise exceptions.SystemSetupError(
-                "You cannot specify any plugins for Docker clusters. Please remove " '"--elasticsearch-plugins" and try again.'
+                'You cannot specify any plugins for Docker clusters. Please remove "--elasticsearch-plugins" and try again.'
             )
         p = provisioner.docker(cfg=cfg, car=car, ip=ip, http_port=http_port, target_root=root_path, node_name=node_name)
         # there is no binary for Docker that can be downloaded / built upfront
@@ -667,9 +667,12 @@ def create(
     elif docker:
         if len(plugins) > 0:
             raise exceptions.SystemSetupError(
-                "You cannot specify any plugins for Docker clusters. Please remove " '"--elasticsearch-plugins" and try again.'
+                'You cannot specify any plugins for Docker clusters. Please remove "--elasticsearch-plugins" and try again.'
             )
-        s = lambda: None
+
+        def s():
+            return None
+
         p = []
         for node_id in node_ids:
             node_name = "%s-%s" % (node_name_prefix, node_id)

esrally/mechanic/supplier.py

@@ -39,7 +39,7 @@ def create(cfg, sources, distribution, car, plugins=None):
     revisions = _extract_revisions(cfg.opts("mechanic", "source.revision", mandatory=sources))
     distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False)
     supply_requirements = _supply_requirements(sources, distribution, plugins, revisions, distribution_version)
-    build_needed = any([build for _, _, build in supply_requirements.values()])
+    build_needed = any(build for _, _, build in supply_requirements.values())
     es_supplier_type, es_version, _ = supply_requirements["elasticsearch"]
     src_config = cfg.all_opts("source")
     suppliers = []
@@ -142,8 +142,7 @@ def _required_version(version):
             "Could not determine version. Please specify the Elasticsearch distribution "
             "to download with the command line parameter --distribution-version."
         )
-    else:
-        return version
+    return version
 
 
 def _required_revision(revisions, key, name=None):
@@ -183,10 +182,9 @@ def _supply_requirements(sources, distribution, plugins, revisions, distribution
                     plugin_revision = revisions.get("all")
                     if not plugin_revision or SourceRepository.is_commit_hash(plugin_revision):
                         raise exceptions.SystemSetupError("No revision specified for plugin [%s]." % plugin.name)
-                    else:
-                        logging.getLogger(__name__).info(
-                            "Revision for [%s] is not explicitly defined. Using catch-all revision [%s].", plugin.name, plugin_revision
-                        )
+                    logging.getLogger(__name__).info(
+                        "Revision for [%s] is not explicitly defined. Using catch-all revision [%s].", plugin.name, plugin_revision
+                    )
                 supply_requirements[plugin.name] = ("source", plugin_revision, True)
             else:
                 supply_requirements[plugin.name] = (distribution, _required_version(distribution_version), False)
@@ -425,7 +423,8 @@ def __init__(self, plugin, revision, src_dir, src_config, builder):
         dir_cfg_key = "plugin.%s.src.dir" % self.plugin.name
         if dir_cfg_key in self.src_config and subdir_cfg_key in self.src_config:
             raise exceptions.SystemSetupError("Can only specify one of %s and %s but both are set." % (dir_cfg_key, subdir_cfg_key))
-        elif dir_cfg_key in self.src_config:
+
+        if dir_cfg_key in self.src_config:
             self.plugin_src_dir = _config_value(self.src_config, dir_cfg_key)
             # we must build directly in the plugin dir, not relative to Elasticsearch
             self.override_build_dir = self.plugin_src_dir
@@ -734,8 +733,7 @@ def _url_for(self, user_defined_key, default_key, mandatory=True):
         except KeyError:
             if mandatory:
                 raise exceptions.SystemSetupError("Neither config key [{}] nor [{}] is defined.".format(user_defined_key, default_key))
-            else:
-                return None
+            return None
         return self.template_renderer.render(url_template)
 
     @property

esrally/mechanic/team.py

@@ -374,11 +374,10 @@ def load_plugin(self, name, config_names, plugin_params=None):
                             "and configurations with %s list elasticsearch-plugins "
                             "--distribution-version=VERSION." % (name, config_name, PROGRAM_NAME)
                         )
-                    else:
-                        raise exceptions.SystemSetupError(
-                            "Unknown plugin [%s]. List the available plugins with %s list "
-                            "elasticsearch-plugins --distribution-version=VERSION." % (name, PROGRAM_NAME)
-                        )
+                    raise exceptions.SystemSetupError(
+                        "Unknown plugin [%s]. List the available plugins with %s list "
+                        "elasticsearch-plugins --distribution-version=VERSION." % (name, PROGRAM_NAME)
+                    )
 
                 config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation())
                 # Do not modify the case of option keys but read them as is

esrally/metrics.py

@@ -1722,7 +1722,8 @@ def percentiles_for_sample_size(sample_size):
     # if needed we can come up with something smarter but it'll do for now
     if sample_size < 1:
         raise AssertionError("Percentiles require at least one sample")
-    elif sample_size == 1:
+
+    if sample_size == 1:
         return [100]
     elif 1 < sample_size < 10:
         return [50, 100]

esrally/rally.py

@@ -834,7 +834,8 @@ def with_actor_system(runnable, cfg):
                 raise exceptions.UserInterrupted(
                     f"User has cancelled the benchmark (shutdown not complete as user interrupted " f"{times_interrupted} times)."
                 ) from None
-            elif not shutdown_complete:
+
+            if not shutdown_complete:
                 console.warn(
                     "Could not terminate all internal processes within timeout. Please check and force-terminate all Rally processes."
                 )

esrally/telemetry.py

@@ -874,7 +874,7 @@ def iterate():
         if stats:
             return dict(iterate())
         else:
-            return dict()
+            return {}
 
     def indices_stats(self, node_name, node_stats, include):
         idx_stats = node_stats["indices"]