From 4a55b906832e9cfc4298a1699061d08405c67b75 Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Wed, 24 Aug 2022 17:38:18 +0400 Subject: [PATCH 1/2] Fix `make lint` on Python 3.10 In practice this means upgrading pylint which was failing on 3.9 and 3.10. --- .pylintrc | 105 +++------------------------ benchmarks/track/bulk_params_test.py | 24 ++---- esrally/client/factory.py | 16 ++-- esrally/driver/runner.py | 9 ++- esrally/mechanic/mechanic.py | 9 ++- esrally/mechanic/supplier.py | 18 ++--- esrally/mechanic/team.py | 9 +-- esrally/metrics.py | 3 +- esrally/rally.py | 3 +- esrally/telemetry.py | 2 +- esrally/track/loader.py | 77 ++++++++++---------- esrally/track/params.py | 14 ++-- esrally/utils/io.py | 2 + esrally/utils/repo.py | 7 +- pyproject.toml | 2 +- tests/driver/driver_test.py | 2 +- tests/driver/runner_test.py | 2 +- tests/mechanic/java_resolver_test.py | 2 +- tests/mechanic/mechanic_test.py | 6 +- tests/mechanic/provisioner_test.py | 2 +- tests/mechanic/supplier_test.py | 2 +- tests/metrics_test.py | 2 +- tests/racecontrol_test.py | 2 +- tests/telemetry_test.py | 6 +- tests/track/loader_test.py | 2 +- tests/utils/console_test.py | 2 +- tests/utils/git_test.py | 2 +- tests/utils/io_test.py | 2 +- tests/utils/jvm_test.py | 2 +- tests/utils/net_test.py | 2 +- tests/utils/process_test.py | 5 +- tests/utils/repo_test.py | 2 +- tests/utils/versions_test.py | 4 +- 33 files changed, 131 insertions(+), 218 deletions(-) diff --git a/.pylintrc b/.pylintrc index 2d1a4910a..4d4a1b679 100644 --- a/.pylintrc +++ b/.pylintrc @@ -46,11 +46,6 @@ suggestion-mode=yes # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no -# pylint-quotes -string-quote=double -triple-quote=double -docstring-quote=double - [MESSAGES CONTROL] # Only show warnings with the listed confidence levels. Leave empty to show @@ -66,17 +61,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, +disable=raw-checker-failed, bad-inline-option, locally-disabled, file-ignored, @@ -84,67 +69,6 @@ disable=print-statement, useless-suppression, deprecated-pragma, use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, # custom rally ignores start here invalid-name, missing-module-docstring, @@ -153,27 +77,27 @@ disable=print-statement, too-many-arguments, too-many-locals, fixme, - no-self-use, too-many-branches, too-few-public-methods, redefined-builtin, global-statement, broad-except, no-else-return, - no-else-raise, no-else-continue, no-member, too-many-instance-attributes, too-many-statements, inconsistent-return-statements, - C0302, - C4001, - R0916, - W0201, - W0613, - W0621, - invalid-docstring-quote, - raise-missing-from + raise-missing-from, + unspecified-encoding, + consider-using-dict-items, + too-many-lines, + too-many-boolean-expressions, + attribute-defined-outside-init, + unused-argument, + redefined-outer-name, + consider-using-f-string, # pyupgrade will cover that + use-implicit-booleaness-not-comparison, # Enable the message, report, category or checker with the given id(s). You can @@ -365,13 +289,6 @@ max-line-length=140 # Maximum number of lines in a module. max-module-lines=1000 -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator - # Allow the body of a class to be on the same line as the declaration if body # contains single statement. single-line-class-stmt=no diff --git a/benchmarks/track/bulk_params_test.py b/benchmarks/track/bulk_params_test.py index 90cf11450..6aab90729 100644 --- a/benchmarks/track/bulk_params_test.py +++ b/benchmarks/track/bulk_params_test.py @@ -87,10 +87,8 @@ def create_reader(bulk_size): disable_gc=True, ) def test_index_data_reader_100(benchmark): - reader = create_reader(bulk_size=100) - reader.__enter__() - benchmark(reader.__next__) - reader.__exit__(None, None, None) + with create_reader(bulk_size=100) as reader: + benchmark(reader.__next__) @pytest.mark.benchmark( @@ -100,10 +98,8 @@ def test_index_data_reader_100(benchmark): disable_gc=True, ) def test_index_data_reader_1000(benchmark): - reader = create_reader(bulk_size=1000) - reader.__enter__() - benchmark(reader.__next__) - reader.__exit__(None, None, None) + with create_reader(bulk_size=1000) as reader: + benchmark(reader.__next__) @pytest.mark.benchmark( @@ -113,10 +109,8 @@ def test_index_data_reader_1000(benchmark): disable_gc=True, ) def test_index_data_reader_10000(benchmark): - reader = create_reader(bulk_size=10000) - reader.__enter__() - benchmark(reader.__next__) - reader.__exit__(None, None, None) + with create_reader(bulk_size=10000) as reader: + benchmark(reader.__next__) @pytest.mark.benchmark( @@ -126,7 +120,5 @@ def test_index_data_reader_10000(benchmark): disable_gc=True, ) def test_index_data_reader_100000(benchmark): - reader = create_reader(bulk_size=100000) - reader.__enter__() - benchmark(reader.__next__) - reader.__exit__(None, None, None) + with create_reader(bulk_size=100000) as reader: + benchmark(reader.__next__) diff --git a/esrally/client/factory.py b/esrally/client/factory.py index 334c575b8..e240b8c0e 100644 --- a/esrally/client/factory.py +++ b/esrally/client/factory.py @@ -126,8 +126,7 @@ def __init__(self, hosts, client_options): "to 'create_api_key_per_client' in order to create client API keys." ) ) - else: - self.logger.info("Automatic creation of client API keys: on") + self.logger.info("Automatic creation of client API keys: on") else: self.logger.info("Automatic creation of client API keys: off") @@ -254,9 +253,8 @@ def wait_for_rest_layer(es, max_attempts=40): except elasticsearch.ConnectionError as e: if "SSL: UNKNOWN_PROTOCOL" in str(e): raise exceptions.SystemSetupError("Could not connect to cluster via https. Is this an https endpoint?", e) - else: - logger.debug("Got connection error on attempt [%s]. Sleeping...", attempt) - time.sleep(3) + logger.debug("Got connection error on attempt [%s]. Sleeping...", attempt) + time.sleep(3) except elasticsearch.TransportError as e: # cluster block, x-pack not initialized yet, our wait condition is not reached if e.status_code in (503, 401, 408): @@ -292,9 +290,8 @@ def create_api_key(es, client_id, max_attempts=5): raise exceptions.SystemSetupError( "Got status code 405 when attempting to create API keys. Is Elasticsearch Security enabled?", e ) - else: - logger.debug("Got status code [%s] on attempt [%s] of [%s]. Sleeping...", e.status_code, attempt, max_attempts) - time.sleep(1) + logger.debug("Got status code [%s] on attempt [%s] of [%s]. Sleeping...", e.status_code, attempt, max_attempts) + time.sleep(1) def delete_api_keys(es, ids, max_attempts=5): @@ -312,8 +309,7 @@ def raise_exception(failed_ids, cause=None): msg = f"Could not delete API keys with the following IDs: {failed_ids}" if cause is not None: raise exceptions.RallyError(msg) from cause - else: - raise exceptions.RallyError(msg) + raise exceptions.RallyError(msg) # Before ES 7.10, deleting API keys by ID had to be done individually. # After ES 7.10, a list of API key IDs can be deleted in one request. diff --git a/esrally/driver/runner.py b/esrally/driver/runner.py index aa2c54d54..c20b8419c 100644 --- a/esrally/driver/runner.py +++ b/esrally/driver/runner.py @@ -2274,7 +2274,8 @@ async def __call__(self, es, params): if state == "failed": failure_reason = stats_response["transforms"][0].get("reason", "unknown") raise exceptions.RallyAssertionError(f"Transform [{transform_id}] failed with [{failure_reason}].") - elif state == "stopped" or wait_for_completion is False: + + if state == "stopped" or wait_for_completion is False: self._completed = True self._percent_completed = 1.0 else: @@ -2813,12 +2814,12 @@ async def __call__(self, es, params): except (socket.timeout, elasticsearch.exceptions.ConnectionError): if last_attempt or not retry_on_timeout: raise - else: - await asyncio.sleep(sleep_time) + await asyncio.sleep(sleep_time) except elasticsearch.exceptions.TransportError as e: if last_attempt or not retry_on_timeout: raise e - elif e.status_code == 408: + + if e.status_code == 408: self.logger.info("[%s] has timed out. Retrying in [%.2f] seconds.", repr(self.delegate), sleep_time) await asyncio.sleep(sleep_time) else: diff --git a/esrally/mechanic/mechanic.py b/esrally/mechanic/mechanic.py index 5cdc9acb1..b930c61d2 100644 --- a/esrally/mechanic/mechanic.py +++ b/esrally/mechanic/mechanic.py @@ -72,7 +72,7 @@ def install(cfg): elif build_type == "docker": if len(plugins) > 0: raise exceptions.SystemSetupError( - "You cannot specify any plugins for Docker clusters. Please remove " '"--elasticsearch-plugins" and try again.' + 'You cannot specify any plugins for Docker clusters. Please remove "--elasticsearch-plugins" and try again.' ) p = provisioner.docker(cfg=cfg, car=car, ip=ip, http_port=http_port, target_root=root_path, node_name=node_name) # there is no binary for Docker that can be downloaded / built upfront @@ -667,9 +667,12 @@ def create( elif docker: if len(plugins) > 0: raise exceptions.SystemSetupError( - "You cannot specify any plugins for Docker clusters. Please remove " '"--elasticsearch-plugins" and try again.' + 'You cannot specify any plugins for Docker clusters. Please remove "--elasticsearch-plugins" and try again.' ) - s = lambda: None + + def s(): + return None + p = [] for node_id in node_ids: node_name = "%s-%s" % (node_name_prefix, node_id) diff --git a/esrally/mechanic/supplier.py b/esrally/mechanic/supplier.py index 7380d9d37..101471847 100644 --- a/esrally/mechanic/supplier.py +++ b/esrally/mechanic/supplier.py @@ -39,7 +39,7 @@ def create(cfg, sources, distribution, car, plugins=None): revisions = _extract_revisions(cfg.opts("mechanic", "source.revision", mandatory=sources)) distribution_version = cfg.opts("mechanic", "distribution.version", mandatory=False) supply_requirements = _supply_requirements(sources, distribution, plugins, revisions, distribution_version) - build_needed = any([build for _, _, build in supply_requirements.values()]) + build_needed = any(build for _, _, build in supply_requirements.values()) es_supplier_type, es_version, _ = supply_requirements["elasticsearch"] src_config = cfg.all_opts("source") suppliers = [] @@ -142,8 +142,7 @@ def _required_version(version): "Could not determine version. Please specify the Elasticsearch distribution " "to download with the command line parameter --distribution-version." ) - else: - return version + return version def _required_revision(revisions, key, name=None): @@ -183,10 +182,9 @@ def _supply_requirements(sources, distribution, plugins, revisions, distribution plugin_revision = revisions.get("all") if not plugin_revision or SourceRepository.is_commit_hash(plugin_revision): raise exceptions.SystemSetupError("No revision specified for plugin [%s]." % plugin.name) - else: - logging.getLogger(__name__).info( - "Revision for [%s] is not explicitly defined. Using catch-all revision [%s].", plugin.name, plugin_revision - ) + logging.getLogger(__name__).info( + "Revision for [%s] is not explicitly defined. Using catch-all revision [%s].", plugin.name, plugin_revision + ) supply_requirements[plugin.name] = ("source", plugin_revision, True) else: supply_requirements[plugin.name] = (distribution, _required_version(distribution_version), False) @@ -425,7 +423,8 @@ def __init__(self, plugin, revision, src_dir, src_config, builder): dir_cfg_key = "plugin.%s.src.dir" % self.plugin.name if dir_cfg_key in self.src_config and subdir_cfg_key in self.src_config: raise exceptions.SystemSetupError("Can only specify one of %s and %s but both are set." % (dir_cfg_key, subdir_cfg_key)) - elif dir_cfg_key in self.src_config: + + if dir_cfg_key in self.src_config: self.plugin_src_dir = _config_value(self.src_config, dir_cfg_key) # we must build directly in the plugin dir, not relative to Elasticsearch self.override_build_dir = self.plugin_src_dir @@ -734,8 +733,7 @@ def _url_for(self, user_defined_key, default_key, mandatory=True): except KeyError: if mandatory: raise exceptions.SystemSetupError("Neither config key [{}] nor [{}] is defined.".format(user_defined_key, default_key)) - else: - return None + return None return self.template_renderer.render(url_template) @property diff --git a/esrally/mechanic/team.py b/esrally/mechanic/team.py index fed840e57..f46261f43 100644 --- a/esrally/mechanic/team.py +++ b/esrally/mechanic/team.py @@ -374,11 +374,10 @@ def load_plugin(self, name, config_names, plugin_params=None): "and configurations with %s list elasticsearch-plugins " "--distribution-version=VERSION." % (name, config_name, PROGRAM_NAME) ) - else: - raise exceptions.SystemSetupError( - "Unknown plugin [%s]. List the available plugins with %s list " - "elasticsearch-plugins --distribution-version=VERSION." % (name, PROGRAM_NAME) - ) + raise exceptions.SystemSetupError( + "Unknown plugin [%s]. List the available plugins with %s list " + "elasticsearch-plugins --distribution-version=VERSION." % (name, PROGRAM_NAME) + ) config = configparser.ConfigParser(interpolation=configparser.ExtendedInterpolation()) # Do not modify the case of option keys but read them as is diff --git a/esrally/metrics.py b/esrally/metrics.py index cfd670de8..3e46cf338 100644 --- a/esrally/metrics.py +++ b/esrally/metrics.py @@ -1722,7 +1722,8 @@ def percentiles_for_sample_size(sample_size): # if needed we can come up with something smarter but it'll do for now if sample_size < 1: raise AssertionError("Percentiles require at least one sample") - elif sample_size == 1: + + if sample_size == 1: return [100] elif 1 < sample_size < 10: return [50, 100] diff --git a/esrally/rally.py b/esrally/rally.py index c47563086..898d02dcd 100644 --- a/esrally/rally.py +++ b/esrally/rally.py @@ -834,7 +834,8 @@ def with_actor_system(runnable, cfg): raise exceptions.UserInterrupted( f"User has cancelled the benchmark (shutdown not complete as user interrupted " f"{times_interrupted} times)." ) from None - elif not shutdown_complete: + + if not shutdown_complete: console.warn( "Could not terminate all internal processes within timeout. Please check and force-terminate all Rally processes." ) diff --git a/esrally/telemetry.py b/esrally/telemetry.py index 81bb9eb61..44ec2e387 100644 --- a/esrally/telemetry.py +++ b/esrally/telemetry.py @@ -874,7 +874,7 @@ def iterate(): if stats: return dict(iterate()) else: - return dict() + return {} def indices_stats(self, node_name, node_stats, include): idx_stats = node_stats["indices"] diff --git a/esrally/track/loader.py b/esrally/track/loader.py index 724efea56..6d7e8eb89 100644 --- a/esrally/track/loader.py +++ b/esrally/track/loader.py @@ -522,13 +522,13 @@ def download(self, base_url, target_path, size_in_bytes): raise exceptions.DataError( "This track does not support test mode. Ask the track author to add it or disable test mode and retry." ) from None + + msg = f"Could not download [{data_url}] to [{target_path}]" + if e.reason: + msg += f" (HTTP status: {e.code}, reason: {e.reason})" else: - msg = f"Could not download [{data_url}] to [{target_path}]" - if e.reason: - msg += f" (HTTP status: {e.code}, reason: {e.reason})" - else: - msg += f" (HTTP status: {e.code})" - raise exceptions.DataError(msg) from e + msg += f" (HTTP status: {e.code})" + raise exceptions.DataError(msg) from e except urllib.error.URLError as e: raise exceptions.DataError(f"Could not download [{data_url}] to [{target_path}].") from e @@ -614,8 +614,7 @@ def prepare_document_set(self, document_set, data_root): f"size of [{expected_size}] bytes and it cannot be downloaded " f"because no base URL is provided." ) from None - else: - raise + raise self.create_file_offset_table(doc_path, document_set.number_of_lines) @@ -1008,40 +1007,42 @@ def read(self, track_name, track_spec_file, mapping_dir): self.logger.info("Reading track specification file [%s].", track_spec_file) # render the track to a temporary file instead of dumping it into the logs. It is easier to check for error messages # involving lines numbers and it also does not bloat Rally's log file so much. - tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".json") - try: - rendered = render_template_from_file(track_spec_file, self.track_params, complete_track_params=self.complete_track_params) - with open(tmp.name, "wt", encoding="utf-8") as f: - f.write(rendered) - self.logger.info("Final rendered track for '%s' has been written to '%s'.", track_spec_file, tmp.name) - track_spec = json.loads(rendered) - except jinja2.exceptions.TemplateNotFound: - self.logger.exception("Could not load [%s]", track_spec_file) - raise exceptions.SystemSetupError("Track {} does not exist".format(track_name)) - except json.JSONDecodeError as e: - self.logger.exception("Could not load [%s].", track_spec_file) - msg = "Could not load '{}': {}.".format(track_spec_file, str(e)) - if e.doc and e.lineno > 0 and e.colno > 0: - line_idx = e.lineno - 1 - lines = e.doc.split("\n") - ctx_line_count = 3 - ctx_start = max(0, line_idx - ctx_line_count) - ctx_end = min(line_idx + ctx_line_count, len(lines)) - erroneous_lines = lines[ctx_start:ctx_end] - erroneous_lines.insert(line_idx - ctx_start + 1, "-" * (e.colno - 1) + "^ Error is here") - msg += " Lines containing the error:\n\n{}\n\n".format("\n".join(erroneous_lines)) - msg += "The complete track has been written to '{}' for diagnosis.".format(tmp.name) - raise TrackSyntaxError(msg) - except Exception as e: - self.logger.exception("Could not load [%s].", track_spec_file) - msg = "Could not load '{}'. The complete track has been written to '{}' for diagnosis.".format(track_spec_file, tmp.name) - raise TrackSyntaxError(msg, e) + with tempfile.NamedTemporaryFile(delete=False, suffix=".json") as tmp: + try: + rendered = render_template_from_file(track_spec_file, self.track_params, complete_track_params=self.complete_track_params) + with open(tmp.name, "wt", encoding="utf-8") as f: + f.write(rendered) + self.logger.info("Final rendered track for '%s' has been written to '%s'.", track_spec_file, tmp.name) + track_spec = json.loads(rendered) + except jinja2.exceptions.TemplateNotFound: + self.logger.exception("Could not load [%s]", track_spec_file) + raise exceptions.SystemSetupError("Track {} does not exist".format(track_name)) + except json.JSONDecodeError as e: + self.logger.exception("Could not load [%s].", track_spec_file) + msg = "Could not load '{}': {}.".format(track_spec_file, str(e)) + if e.doc and e.lineno > 0 and e.colno > 0: + line_idx = e.lineno - 1 + lines = e.doc.split("\n") + ctx_line_count = 3 + ctx_start = max(0, line_idx - ctx_line_count) + ctx_end = min(line_idx + ctx_line_count, len(lines)) + erroneous_lines = lines[ctx_start:ctx_end] + erroneous_lines.insert(line_idx - ctx_start + 1, "-" * (e.colno - 1) + "^ Error is here") + msg += " Lines containing the error:\n\n{}\n\n".format("\n".join(erroneous_lines)) + msg += "The complete track has been written to '{}' for diagnosis.".format(tmp.name) + raise TrackSyntaxError(msg) + except Exception as e: + self.logger.exception("Could not load [%s].", track_spec_file) + msg = "Could not load '{}'. The complete track has been written to '{}' for diagnosis.".format(track_spec_file, tmp.name) + raise TrackSyntaxError(msg, e) + # check the track version before even attempting to validate the JSON format to avoid bogus errors. raw_version = track_spec.get("version", TrackFileReader.MAXIMUM_SUPPORTED_TRACK_VERSION) try: track_version = int(raw_version) except ValueError: raise exceptions.InvalidSyntax("version identifier for track %s must be numeric but was [%s]" % (track_name, str(raw_version))) + if TrackFileReader.MINIMUM_SUPPORTED_TRACK_VERSION > track_version: raise exceptions.RallyError( "Track {} is on version {} but needs to be updated at least to version {} to work with the " @@ -1052,6 +1053,7 @@ def read(self, track_name, track_spec_file, mapping_dir): "Track {} requires a newer version of Rally. Please upgrade Rally (supported track version: {}, " "required track version: {}).".format(track_name, TrackFileReader.MAXIMUM_SUPPORTED_TRACK_VERSION, track_version) ) + try: jsonschema.validate(track_spec, self.track_schema) except jsonschema.exceptions.ValidationError as ve: @@ -1361,7 +1363,8 @@ def _create_corpora(self, corpora_specs, indices, data_streams): if target_ds and len(indices) > 0: # if indices are in use we error raise TrackSyntaxError("target-data-stream cannot be used when using indices") - elif target_ds and target_type: + + if target_ds and target_type: raise TrackSyntaxError("target-type cannot be used when using data-streams") # need an index if we're using indices and no meta-data are present and we don't have a default diff --git a/esrally/track/params.py b/esrally/track/params.py index 689848ef9..d9f222a20 100644 --- a/esrally/track/params.py +++ b/esrally/track/params.py @@ -1200,13 +1200,13 @@ def __iter__(self): def __next__(self): if self.current_line >= self.number_of_lines: raise StopIteration() - else: - # ensure we don't read past the allowed number of lines. - lines = self.source.readlines(min(self.bulk_size, self.number_of_lines - self.current_line)) - self.current_line += len(lines) - if len(lines) == 0: - raise StopIteration() - return lines + + # ensure we don't read past the allowed number of lines. + lines = self.source.readlines(min(self.bulk_size, self.number_of_lines - self.current_line)) + self.current_line += len(lines) + if len(lines) == 0: + raise StopIteration() + return lines def __str__(self): return "%s[%d;%d]" % (self.source, self.offset, self.offset + self.number_of_lines) diff --git a/esrally/utils/io.py b/esrally/utils/io.py index 4b2e666dc..b8c50351a 100644 --- a/esrally/utils/io.py +++ b/esrally/utils/io.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +# pylint: disable=consider-using-with + import bz2 import gzip import logging diff --git a/esrally/utils/repo.py b/esrally/utils/repo.py index 966ae7528..70be0b282 100644 --- a/esrally/utils/repo.py +++ b/esrally/utils/repo.py @@ -55,10 +55,9 @@ def __init__(self, remote_url, root_dir, repo_name, resource_name, offline, fetc raise exceptions.SystemSetupError( "[{src}] must be a git repository.\n\nPlease run:\ngit -C {src} init".format(src=self.repo_dir) ) - else: - raise exceptions.SystemSetupError( - "Expected a git repository at [{src}] but the directory does not exist.".format(src=self.repo_dir) - ) + raise exceptions.SystemSetupError( + "Expected a git repository at [{src}] but the directory does not exist.".format(src=self.repo_dir) + ) def update(self, distribution_version): try: diff --git a/pyproject.toml b/pyproject.toml index 2a38ecf0c..1e17fd2db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,7 +100,7 @@ develop = [ "wheel==0.37.1", "github3.py==1.3.0", "pre-commit==2.20.0", - "pylint==2.6.0", + "pylint==2.14.5", "trustme==0.9.0", "pytest-httpserver==1.0.4", ] diff --git a/tests/driver/driver_test.py b/tests/driver/driver_test.py index 295111ae7..cdbdd4e88 100644 --- a/tests/driver/driver_test.py +++ b/tests/driver/driver_test.py @@ -20,8 +20,8 @@ import io import threading import time -import unittest.mock as mock from datetime import datetime +from unittest import mock import elasticsearch import pytest diff --git a/tests/driver/runner_test.py b/tests/driver/runner_test.py index d22b2f238..0e251f922 100644 --- a/tests/driver/runner_test.py +++ b/tests/driver/runner_test.py @@ -22,7 +22,7 @@ import json import math import random -import unittest.mock as mock +from unittest import mock import elasticsearch import pytest diff --git a/tests/mechanic/java_resolver_test.py b/tests/mechanic/java_resolver_test.py index 7aba6d0ca..796dc4b0d 100644 --- a/tests/mechanic/java_resolver_test.py +++ b/tests/mechanic/java_resolver_test.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -import unittest.mock as mock +from unittest import mock import pytest diff --git a/tests/mechanic/mechanic_test.py b/tests/mechanic/mechanic_test.py index fa162fd88..4c106790a 100644 --- a/tests/mechanic/mechanic_test.py +++ b/tests/mechanic/mechanic_test.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -import unittest.mock as mock +from unittest import mock import pytest @@ -115,7 +115,9 @@ def _add_results(self, current_race, node): @mock.patch("esrally.mechanic.provisioner.cleanup") def test_start_stop_nodes(self, cleanup): - supplier = lambda: "/home/user/src/elasticsearch/es.tar.gz" + def supplier(): + return "/home/user/src/elasticsearch/es.tar.gz" + provisioners = [mock.Mock(), mock.Mock()] launcher = self.MockLauncher() cfg = config.Config() diff --git a/tests/mechanic/provisioner_test.py b/tests/mechanic/provisioner_test.py index 083a737db..e7c3fc5f2 100644 --- a/tests/mechanic/provisioner_test.py +++ b/tests/mechanic/provisioner_test.py @@ -18,7 +18,7 @@ import os import tempfile -import unittest.mock as mock +from unittest import mock import pytest diff --git a/tests/mechanic/supplier_test.py b/tests/mechanic/supplier_test.py index 1f9bcb987..6dd787c17 100644 --- a/tests/mechanic/supplier_test.py +++ b/tests/mechanic/supplier_test.py @@ -18,7 +18,7 @@ import collections import datetime -import unittest.mock as mock +from unittest import mock import pytest diff --git a/tests/metrics_test.py b/tests/metrics_test.py index 9a8b4e885..9a5073771 100644 --- a/tests/metrics_test.py +++ b/tests/metrics_test.py @@ -23,8 +23,8 @@ import random import string import tempfile -import unittest.mock as mock import uuid +from unittest import mock import elasticsearch.exceptions import pytest diff --git a/tests/racecontrol_test.py b/tests/racecontrol_test.py index e30b54f17..1f4a4a7f1 100644 --- a/tests/racecontrol_test.py +++ b/tests/racecontrol_test.py @@ -17,7 +17,7 @@ import os import re -import unittest.mock as mock +from unittest import mock import pytest diff --git a/tests/telemetry_test.py b/tests/telemetry_test.py index 440cc37d6..e43f8b799 100644 --- a/tests/telemetry_test.py +++ b/tests/telemetry_test.py @@ -19,8 +19,8 @@ import copy import logging import random -import unittest.mock as mock from collections import namedtuple +from unittest import mock from unittest.mock import call import elasticsearch @@ -227,8 +227,7 @@ def perform_request(self, *args, **kwargs): self.kwargs.append(kwargs) if self._force_error: raise self._error - else: - return copy.deepcopy(self._response) + return copy.deepcopy(self._response) class TestJfr: @@ -352,6 +351,7 @@ def test_generates_log_file(self, file_mock, es): segment_stats = telemetry.SegmentStats("/var/log", es) segment_stats.on_benchmark_stop() es.cat.segments.assert_called_with(index="_all", v=True) + # pylint: disable=unnecessary-dunder-call file_mock.assert_has_calls( [ call("/var/log/segment_stats.log", "wt"), diff --git a/tests/track/loader_test.py b/tests/track/loader_test.py index 3f278a7bc..0e19242a5 100644 --- a/tests/track/loader_test.py +++ b/tests/track/loader_test.py @@ -19,8 +19,8 @@ import random import re import textwrap -import unittest.mock as mock import urllib.error +from unittest import mock import pytest diff --git a/tests/utils/console_test.py b/tests/utils/console_test.py index 74c2ac949..8ba3df8c0 100644 --- a/tests/utils/console_test.py +++ b/tests/utils/console_test.py @@ -17,7 +17,7 @@ import os import random -import unittest.mock as mock +from unittest import mock import pytest diff --git a/tests/utils/git_test.py b/tests/utils/git_test.py index 502dea0c9..97cbe02b9 100644 --- a/tests/utils/git_test.py +++ b/tests/utils/git_test.py @@ -17,7 +17,7 @@ import logging import os -import unittest.mock as mock +from unittest import mock import pytest diff --git a/tests/utils/io_test.py b/tests/utils/io_test.py index b511458d7..0f1133f74 100644 --- a/tests/utils/io_test.py +++ b/tests/utils/io_test.py @@ -20,7 +20,7 @@ import os import subprocess import tempfile -import unittest.mock as mock +from unittest import mock from esrally.utils import io diff --git a/tests/utils/jvm_test.py b/tests/utils/jvm_test.py index e8c94bbd6..f46c74dd0 100644 --- a/tests/utils/jvm_test.py +++ b/tests/utils/jvm_test.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -import unittest.mock as mock +from unittest import mock import pytest diff --git a/tests/utils/net_test.py b/tests/utils/net_test.py index 07764eb47..531ccb160 100644 --- a/tests/utils/net_test.py +++ b/tests/utils/net_test.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. import random -import unittest.mock as mock +from unittest import mock import pytest diff --git a/tests/utils/process_test.py b/tests/utils/process_test.py index 095f79a7b..b54916d71 100644 --- a/tests/utils/process_test.py +++ b/tests/utils/process_test.py @@ -16,7 +16,7 @@ # under the License. import os -import unittest.mock as mock +from unittest import mock import psutil @@ -42,8 +42,7 @@ def kill(self): def status(self): if self.killed: raise psutil.NoSuchProcess(self.pid) - else: - return "running" + return "running" class TestProcess: diff --git a/tests/utils/repo_test.py b/tests/utils/repo_test.py index 427a0d1b0..fdc98c391 100644 --- a/tests/utils/repo_test.py +++ b/tests/utils/repo_test.py @@ -16,7 +16,7 @@ # under the License. import random -import unittest.mock as mock +from unittest import mock import pytest diff --git a/tests/utils/versions_test.py b/tests/utils/versions_test.py index a38410efb..316d7faf0 100644 --- a/tests/utils/versions_test.py +++ b/tests/utils/versions_test.py @@ -77,7 +77,7 @@ def test_latest_bounded_minor(self, seed): def test_components_ignores_invalid_versions(self): with pytest.raises( exceptions.InvalidSyntax, - match=re.escape(r"version string '5.0.0a' does not conform to pattern " r"'^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$'"), + match=re.escape(r"version string '5.0.0a' does not conform to pattern '^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$'"), ): versions.components("5.0.0a") @@ -94,7 +94,7 @@ def test_versionvariants_parses_correct_version_string(self): def test_versions_rejects_invalid_version_strings(self): with pytest.raises( exceptions.InvalidSyntax, - match=re.escape(r"version string '5.0.0a-SNAPSHOT' does not conform to pattern " r"'^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$'"), + match=re.escape(r"version string '5.0.0a-SNAPSHOT' does not conform to pattern '^(\d+)\.(\d+)\.(\d+)(?:-(.+))?$'"), ): versions.VersionVariants("5.0.0a-SNAPSHOT") From b085d86c72b1e6908fafc742cb22131dd986cf2e Mon Sep 17 00:00:00 2001 From: Quentin Pradet Date: Fri, 26 Aug 2022 14:05:16 +0400 Subject: [PATCH 2/2] Disable duplicate-code pylint check --- .pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.pylintrc b/.pylintrc index 4d4a1b679..0c72f625d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -89,6 +89,7 @@ disable=raw-checker-failed, too-many-statements, inconsistent-return-statements, raise-missing-from, + duplicate-code, unspecified-encoding, consider-using-dict-items, too-many-lines,