From 0e005643280883afe88416c7f3d3425dff8f02b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20=C5=A0ediv=C3=BD?= <6774676+eumiro@users.noreply.github.com> Date: Thu, 24 Aug 2023 22:16:37 +0000 Subject: [PATCH] Use str.splitlines() to split lines (#33592) --- airflow/cli/commands/standalone_command.py | 2 +- airflow/utils/code_utils.py | 28 +++++++++---------- ...assign_cherry_picked_prs_with_milestone.py | 2 +- dev/check_files.py | 2 +- dev/prepare_release_issue.py | 4 +-- .../prepare_provider_packages.py | 2 +- docs/exts/docs_build/code_utils.py | 2 +- docs/exts/docs_build/errors.py | 2 +- docs/exts/docs_build/spelling_checks.py | 2 +- helm_tests/airflow_aux/test_configmap.py | 14 ++-------- kubernetes_tests/test_base.py | 4 +-- tests/cli/commands/test_config_command.py | 24 ++++++++-------- tests/cli/commands/test_dag_command.py | 2 +- tests/models/test_dag.py | 2 +- tests/test_utils/terraform.py | 2 +- 15 files changed, 41 insertions(+), 53 deletions(-) diff --git a/airflow/cli/commands/standalone_command.py b/airflow/cli/commands/standalone_command.py index 0beacb71d159d..28632abf6acfc 100644 --- a/airflow/cli/commands/standalone_command.py +++ b/airflow/cli/commands/standalone_command.py @@ -140,7 +140,7 @@ def print_output(self, name: str, output): "standalone": "white", }.get(name, "white") colorised_name = colored("%10s" % name, color) - for line in output.split("\n"): + for line in output.splitlines(): print(f"{colorised_name} | {line.strip()}") def print_error(self, name: str, output): diff --git a/airflow/utils/code_utils.py b/airflow/utils/code_utils.py index f43a6ce41d059..01eb0c8da040a 100644 --- a/airflow/utils/code_utils.py +++ b/airflow/utils/code_utils.py @@ -19,6 +19,7 @@ import functools import inspect import os +from pathlib import Path from typing import Any @@ -61,21 +62,18 @@ def prepare_code_snippet(file_path: str, line_no: int, context_lines_count: int :param context_lines_count: The number of lines that will be cut before and after. :return: str """ - with open(file_path) as text_file: - # Highlight code - code = text_file.read() - code_lines = code.split("\n") - # Prepend line number - code_lines = [ - f">{lno:3} | {line}" if line_no == lno else f"{lno:4} | {line}" - for lno, line in enumerate(code_lines, 1) - ] - # # Cut out the snippet - start_line_no = max(0, line_no - context_lines_count - 1) - end_line_no = line_no + context_lines_count - code_lines = code_lines[start_line_no:end_line_no] - # Join lines - code = "\n".join(code_lines) + code_lines = Path(file_path).read_text().splitlines() + # Prepend line number + code_lines = [ + f">{lno:3} | {line}" if line_no == lno else f"{lno:4} | {line}" + for lno, line in enumerate(code_lines, 1) + ] + # # Cut out the snippet + start_line_no = max(0, line_no - context_lines_count - 1) + end_line_no = line_no + context_lines_count + code_lines = code_lines[start_line_no:end_line_no] + # Join lines + code = "\n".join(code_lines) return code diff --git a/dev/assign_cherry_picked_prs_with_milestone.py b/dev/assign_cherry_picked_prs_with_milestone.py index 7e4dc0f2a0a89..f48ddc188a706 100755 --- a/dev/assign_cherry_picked_prs_with_milestone.py +++ b/dev/assign_cherry_picked_prs_with_milestone.py @@ -226,7 +226,7 @@ def get_changes(verbose: bool, previous_release: str, current_release: str) -> l cwd=SOURCE_DIR_PATH, text=True, ) - return [get_change_from_line(line) for line in change_strings.split("\n")] + return [get_change_from_line(line) for line in change_strings.splitlines()] def update_milestone(r: Repository, pr: PullRequest, m: Milestone): diff --git a/dev/check_files.py b/dev/check_files.py index f50875cc015c5..e1fb029af724d 100644 --- a/dev/check_files.py +++ b/dev/check_files.py @@ -65,7 +65,7 @@ def get_packages() -> list[tuple[str, str]]: # e.g. https://pypi.org/project/apache-airflow-providers-airbyte/3.1.0rc1/ packages = [] - for line in content.split("\n"): + for line in content.splitlines(): if not line: continue name, version = line.rstrip("/").split("/")[-2:] diff --git a/dev/prepare_release_issue.py b/dev/prepare_release_issue.py index a5bd413f8d8d1..43832ea28447e 100755 --- a/dev/prepare_release_issue.py +++ b/dev/prepare_release_issue.py @@ -169,7 +169,7 @@ def get_changes( cwd=SOURCE_DIR_PATH, text=True, ) - return [get_change_from_line(line) for line in change_strings.split("\n")] + return [get_change_from_line(line) for line in change_strings.splitlines()] def render_template( @@ -300,7 +300,7 @@ def generate_issue_content( # GitHub does not have linked issues in PR - but we quite rigorously add Fixes/Closes # Relate so we can find those from the body if pr.body: - body = pr.body.replace("\n", " ").replace("\r", " ") + body = " ".join(pr.body.splitlines()) linked_issue_numbers = { int(issue_match.group(1)) for issue_match in ISSUE_MATCH_IN_BODY.finditer(body) } diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index b7748c2db84ce..b3d67abc0a473 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -482,7 +482,7 @@ def convert_git_changes_to_table( """ from tabulate import tabulate - lines = changes.split("\n") + lines = changes.splitlines() headers = ["Commit", "Committed", "Subject"] table_data = [] changes_list: list[Change] = [] diff --git a/docs/exts/docs_build/code_utils.py b/docs/exts/docs_build/code_utils.py index 1f520553b84b6..c7eef91c7d993 100644 --- a/docs/exts/docs_build/code_utils.py +++ b/docs/exts/docs_build/code_utils.py @@ -66,7 +66,7 @@ def guess_lexer_for_filename(filename): code=code, formatter=TerminalFormatter(), lexer=guess_lexer_for_filename(file_path) ) - code_lines = code.split("\n") + code_lines = code.splitlines() # Prepend line number code_lines = [f"{line_no:4} | {line}" for line_no, line in enumerate(code_lines, 1)] # # Cut out the snippet diff --git a/docs/exts/docs_build/errors.py b/docs/exts/docs_build/errors.py index 321379896ca87..0abfc8fe573f3 100644 --- a/docs/exts/docs_build/errors.py +++ b/docs/exts/docs_build/errors.py @@ -95,7 +95,7 @@ def parse_sphinx_warnings(warning_text: str, docs_dir: str) -> list[DocBuildErro :return: list of DocBuildErrors. """ sphinx_build_errors = [] - for sphinx_warning in warning_text.split("\n"): + for sphinx_warning in warning_text.splitlines(): if not sphinx_warning: continue warning_parts = sphinx_warning.split(":", 2) diff --git a/docs/exts/docs_build/spelling_checks.py b/docs/exts/docs_build/spelling_checks.py index bbaa9fa5dde79..2e3f7d409f741 100644 --- a/docs/exts/docs_build/spelling_checks.py +++ b/docs/exts/docs_build/spelling_checks.py @@ -90,7 +90,7 @@ def parse_spelling_warnings(warning_text: str, docs_dir: str) -> list[SpellingEr :return: list of SpellingError. """ sphinx_spelling_errors = [] - for sphinx_warning in warning_text.split("\n"): + for sphinx_warning in warning_text.splitlines(): if not sphinx_warning: continue warning_parts = None diff --git a/helm_tests/airflow_aux/test_configmap.py b/helm_tests/airflow_aux/test_configmap.py index b5df4d6237fc7..8d835488cc766 100644 --- a/helm_tests/airflow_aux/test_configmap.py +++ b/helm_tests/airflow_aux/test_configmap.py @@ -124,13 +124,8 @@ def test_default_flower_url_prefix(self): show_only=["templates/configmaps/configmap.yaml"], ) expected = "flower_url_prefix = " - found = False cfg = jmespath.search('data."airflow.cfg"', docs[0]) - for item in cfg.split("\n"): - if item == expected: - found = True - - assert found is True + assert expected in cfg.splitlines() def test_overriden_flower_url_prefix(self): docs = render_chart( @@ -139,11 +134,6 @@ def test_overriden_flower_url_prefix(self): ) expected = "flower_url_prefix = /overriden-path" - found = False cfg = jmespath.search('data."airflow.cfg"', docs[0]) - for item in cfg.split("\n"): - if item == expected: - found = True - - assert found is True + assert expected in cfg.splitlines() diff --git a/kubernetes_tests/test_base.py b/kubernetes_tests/test_base.py index 5b45969c3d287..98c2e978c7480 100644 --- a/kubernetes_tests/test_base.py +++ b/kubernetes_tests/test_base.py @@ -102,7 +102,7 @@ def _describe_resources(self, namespace: str): @staticmethod def _num_pods_in_namespace(namespace): air_pod = check_output(["kubectl", "get", "pods", "-n", namespace]).decode() - air_pod = air_pod.split("\n") + air_pod = air_pod.splitlines() names = [re2.compile(r"\s+").split(x)[0] for x in air_pod if "airflow" in x] return len(names) @@ -110,7 +110,7 @@ def _num_pods_in_namespace(namespace): def _delete_airflow_pod(name=""): suffix = "-" + name if name else "" air_pod = check_output(["kubectl", "get", "pods"]).decode() - air_pod = air_pod.split("\n") + air_pod = air_pod.splitlines() names = [re2.compile(r"\s+").split(x)[0] for x in air_pod if "airflow" + suffix in x] if names: check_call(["kubectl", "delete", "pod", names[0]]) diff --git a/tests/cli/commands/test_config_command.py b/tests/cli/commands/test_config_command.py index c1e2924bfc274..52db863cf66be 100644 --- a/tests/cli/commands/test_config_command.py +++ b/tests/cli/commands/test_config_command.py @@ -78,7 +78,7 @@ def test_cli_show_config_should_only_show_comments_when_no_defaults(self): with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: config_command.show_config(self.parser.parse_args(["config", "list", "--color", "off"])) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() assert all(not line.startswith("#") or line.endswith("= ") for line in lines if line) def test_cli_show_config_shows_descriptions(self): @@ -87,7 +87,7 @@ def test_cli_show_config_shows_descriptions(self): self.parser.parse_args(["config", "list", "--color", "off", "--include-descriptions"]) ) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() # comes from metrics description assert all(not line.startswith("# Source: ") for line in lines if line) assert any(line.startswith("# StatsD") for line in lines if line) @@ -100,7 +100,7 @@ def test_cli_show_config_shows_examples(self): self.parser.parse_args(["config", "list", "--color", "off", "--include-examples"]) ) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() assert all(not line.startswith("# Source: ") for line in lines if line) assert all(not line.startswith("# StatsD") for line in lines if line) assert any(line.startswith("# Example:") for line in lines if line) @@ -112,7 +112,7 @@ def test_cli_show_config_shows_variables(self): self.parser.parse_args(["config", "list", "--color", "off", "--include-env-vars"]) ) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() assert all(not line.startswith("# Source: ") for line in lines if line) assert all(not line.startswith("# StatsD") for line in lines if line) assert all(not line.startswith("# Example:") for line in lines if line) @@ -124,7 +124,7 @@ def test_cli_show_config_shows_sources(self): self.parser.parse_args(["config", "list", "--color", "off", "--include-sources"]) ) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() assert any(line.startswith("# Source: ") for line in lines if line) assert all(not line.startswith("# StatsD") for line in lines if line) assert all(not line.startswith("# Example:") for line in lines if line) @@ -136,7 +136,7 @@ def test_cli_show_config_defaults(self): self.parser.parse_args(["config", "list", "--color", "off", "--defaults"]) ) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() assert all(not line.startswith("# Source: ") for line in lines if line) assert any(line.startswith("# StatsD") for line in lines if line) assert any(not line.startswith("# Example:") for line in lines if line) @@ -151,7 +151,7 @@ def test_cli_show_config_defaults_not_show_conf_changes(self): self.parser.parse_args(["config", "list", "--color", "off", "--defaults"]) ) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() assert any(line.startswith("# task_runner = StandardTaskRunner") for line in lines if line) @mock.patch("os.environ", {"AIRFLOW__CORE__TASK_RUNNER": "test-env-runner"}) @@ -161,7 +161,7 @@ def test_cli_show_config_defaults_do_not_show_env_changes(self): self.parser.parse_args(["config", "list", "--color", "off", "--defaults"]) ) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() assert any(line.startswith("# task_runner = StandardTaskRunner") for line in lines if line) @conf_vars({("core", "task_runner"): "test-runner"}) @@ -169,7 +169,7 @@ def test_cli_show_changed_defaults_when_overridden_in_conf(self): with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: config_command.show_config(self.parser.parse_args(["config", "list", "--color", "off"])) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() assert any(line.startswith("task_runner = test-runner") for line in lines if line) @mock.patch("os.environ", {"AIRFLOW__CORE__TASK_RUNNER": "test-env-runner"}) @@ -177,14 +177,14 @@ def test_cli_show_changed_defaults_when_overridden_in_env(self): with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: config_command.show_config(self.parser.parse_args(["config", "list", "--color", "off"])) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() assert any(line.startswith("task_runner = test-env-runner") for line in lines if line) def test_cli_has_providers(self): with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: config_command.show_config(self.parser.parse_args(["config", "list", "--color", "off"])) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() assert any(line.startswith("celery_config_options") for line in lines if line) def test_cli_comment_out_everything(self): @@ -193,7 +193,7 @@ def test_cli_comment_out_everything(self): self.parser.parse_args(["config", "list", "--color", "off", "--comment-out-everything"]) ) output = temp_stdout.getvalue() - lines = output.split("\n") + lines = output.splitlines() assert all(not line.strip() or line.startswith(("#", "[")) for line in lines if line) diff --git a/tests/cli/commands/test_dag_command.py b/tests/cli/commands/test_dag_command.py index 5cd09cd9c3819..3af7e060e6335 100644 --- a/tests/cli/commands/test_dag_command.py +++ b/tests/cli/commands/test_dag_command.py @@ -679,7 +679,7 @@ def test_trigger_dag_output_as_json(self): with contextlib.redirect_stdout(io.StringIO()) as temp_stdout: dag_command.dag_trigger(args) # get the last line from the logs ignoring all logging lines - out = temp_stdout.getvalue().strip().split("\n")[-1] + out = temp_stdout.getvalue().strip().splitlines()[-1] parsed_out = json.loads(out) assert 1 == len(parsed_out) diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py index 342d08785197e..cc7ad80f185e4 100644 --- a/tests/models/test_dag.py +++ b/tests/models/test_dag.py @@ -1360,7 +1360,7 @@ def test_tree_view(self): dag.tree_view() stdout = stdout.getvalue() - stdout_lines = stdout.split("\n") + stdout_lines = stdout.splitlines() assert "t1" in stdout_lines[0] assert "t2" in stdout_lines[1] assert "t3" in stdout_lines[2] diff --git a/tests/test_utils/terraform.py b/tests/test_utils/terraform.py index 265b8c11ea771..bbb68b60c1a8b 100644 --- a/tests/test_utils/terraform.py +++ b/tests/test_utils/terraform.py @@ -28,7 +28,7 @@ def setup_method(self) -> None: self.execute_cmd(["terraform", "apply", "-input=false", "-auto-approve", self.TERRAFORM_DIR]) def get_tf_output(self, name): - return self.check_output(["terraform", "output", name]).decode("utf-8").replace("\r\n", "") + return "".join(self.check_output(["terraform", "output", name]).decode("utf-8").splitlines()) def teardown_method(self) -> None: self.execute_cmd(["terraform", "plan", "-destroy", "-input=false", self.TERRAFORM_DIR])