From cc5a4364184a6e090a0c6bf6571947ab9e572734 Mon Sep 17 00:00:00 2001 From: Christopher Neugebauer Date: Mon, 6 Feb 2023 07:57:20 -0800 Subject: [PATCH] Adds fields to capture stdout/stderr to `experimental_run_in_sandbox` (#18165) This adds `stdout=` and `stderr=` fields to `experimental_run_in_sandbox`, allowing users to include those streams in the output from such a target. Closes #18087. --- .../pants/backend/shell/target_types.py | 14 +++++++ .../shell/util_rules/adhoc_process_support.py | 31 ++++++++++++--- .../shell/util_rules/run_in_sandbox.py | 9 +++++ .../shell/util_rules/shell_command_test.py | 38 +++++++++++++++++++ 4 files changed, 87 insertions(+), 5 deletions(-) diff --git a/src/python/pants/backend/shell/target_types.py b/src/python/pants/backend/shell/target_types.py index 767897fc1c1..8191157aa46 100644 --- a/src/python/pants/backend/shell/target_types.py +++ b/src/python/pants/backend/shell/target_types.py @@ -389,6 +389,18 @@ class RunInSandboxArgumentsField(StringSequenceField): help = f"Extra arguments to pass into the `{RunInSandboxRunnableField.alias}` field." +class RunInSandboxStdoutFilenameField(StringField): + alias = "stdout" + default = None + help = "A filename to capture the contents of `stdout` to, relative to the value of `workdir`." + + +class RunInSandboxStderrFilenameField(StringField): + alias = "stderr" + default = None + help = "A filename to capture the contents of `stdout` to, relative to the value of `workdir`." + + class ShellCommandTimeoutField(IntField): alias = "timeout" default = 30 @@ -532,6 +544,8 @@ class ShellRunInSandboxTarget(Target): ShellCommandExtraEnvVarsField, ShellCommandWorkdirField, ShellCommandOutputRootDirField, + RunInSandboxStdoutFilenameField, + RunInSandboxStderrFilenameField, EnvironmentField, ) help = softwrap( diff --git a/src/python/pants/backend/shell/util_rules/adhoc_process_support.py b/src/python/pants/backend/shell/util_rules/adhoc_process_support.py index a1c130247ee..5801a8e59dc 100644 --- a/src/python/pants/backend/shell/util_rules/adhoc_process_support.py +++ b/src/python/pants/backend/shell/util_rules/adhoc_process_support.py @@ -10,7 +10,7 @@ import shlex from dataclasses import dataclass from textwrap import dedent # noqa: PNT20 -from typing import Union +from typing import Mapping, Union from pants.backend.shell.subsystems.shell_setup import ShellSetup from pants.backend.shell.target_types import ( @@ -34,7 +34,15 @@ ) from pants.engine.addresses import Addresses, UnparsedAddressInputs from pants.engine.env_vars import EnvironmentVars, EnvironmentVarsRequest -from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, Directory, MergeDigests, Snapshot +from pants.engine.fs import ( + EMPTY_DIGEST, + CreateDigest, + Digest, + Directory, + FileContent, + MergeDigests, + Snapshot, +) from pants.engine.internals.native_engine import RemovePrefix from pants.engine.process import Process from pants.engine.rules import Get, MultiGet, collect_rules, rule, rule_helper @@ -166,7 +174,11 @@ def _parse_outputs_from_command( @rule_helper async def _adjust_root_output_directory( - digest: Digest, address: Address, working_directory: str, root_output_directory: str + digest: Digest, + address: Address, + working_directory: str, + root_output_directory: str, + extra_files: Mapping[str, bytes] = FrozenDict(), ) -> Digest: working_directory = _parse_working_directory(working_directory, address) @@ -174,8 +186,17 @@ async def _adjust_root_output_directory( if new_root == "": return digest - else: - return await Get(Digest, RemovePrefix(digest, new_root)) + + if extra_files: + extra_digest = await Get( + Digest, + CreateDigest( + FileContent(f"{new_root}/{name}", content) for name, content in extra_files.items() + ), + ) + digest = await Get(Digest, MergeDigests((digest, extra_digest))) + + return await Get(Digest, RemovePrefix(digest, new_root)) def _shell_tool_safe_env_name(tool_name: str) -> str: diff --git a/src/python/pants/backend/shell/util_rules/run_in_sandbox.py b/src/python/pants/backend/shell/util_rules/run_in_sandbox.py index d380dc2d311..3debb49fe4a 100644 --- a/src/python/pants/backend/shell/util_rules/run_in_sandbox.py +++ b/src/python/pants/backend/shell/util_rules/run_in_sandbox.py @@ -9,6 +9,8 @@ RunInSandboxArgumentsField, RunInSandboxRunnableField, RunInSandboxSourcesField, + RunInSandboxStderrFilenameField, + RunInSandboxStdoutFilenameField, ShellCommandLogOutputField, ShellCommandOutputRootDirField, ShellCommandWorkdirField, @@ -132,11 +134,18 @@ async def run_in_sandbox_request( if result.stderr: logger.warning(result.stderr.decode()) + extras = ( + (shell_command[RunInSandboxStdoutFilenameField].value, result.stdout), + (shell_command[RunInSandboxStderrFilenameField].value, result.stderr), + ) + extra_contents = {i: j for i, j in extras if i} + adjusted = await _adjust_root_output_directory( result.output_digest, shell_command.address, working_directory, root_output_directory, + extra_files=extra_contents, ) output = await Get(Snapshot, Digest, adjusted) diff --git a/src/python/pants/backend/shell/util_rules/shell_command_test.py b/src/python/pants/backend/shell/util_rules/shell_command_test.py index fb481071d75..eb258e8eedf 100644 --- a/src/python/pants/backend/shell/util_rules/shell_command_test.py +++ b/src/python/pants/backend/shell/util_rules/shell_command_test.py @@ -764,6 +764,44 @@ def test_run_runnable_in_sandbox_with_workdir(rule_runner: RuleRunner) -> None: ) +def test_run_in_sandbox_capture_stdout_err(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "src/fruitcake.py": dedent( + """\ + import sys + print("fruitcake") + print("inconceivable", file=sys.stderr) + """ + ), + "src/BUILD": dedent( + """\ + python_source( + source="fruitcake.py", + name="fruitcake", + ) + + experimental_run_in_sandbox( + name="run_fruitcake", + runnable=":fruitcake", + stdout="stdout", + stderr="stderr", + ) + """ + ), + } + ) + + assert_run_in_sandbox_result( + rule_runner, + Address("src", target_name="run_fruitcake"), + expected_contents={ + "stderr": "inconceivable\n", + "stdout": "fruitcake\n", + }, + ) + + def test_relative_directories(rule_runner: RuleRunner) -> None: rule_runner.write_files( {