From d7777fd6320d7c372447d44c4182258aa5afd3f9 Mon Sep 17 00:00:00 2001 From: Ryan Troost Date: Tue, 27 Feb 2024 15:22:26 -0500 Subject: [PATCH 01/68] fix summaries for actions results (#3174) * fix summaries for actions results * remove negative --- src/Runner.Worker/FileCommandManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index b03c3189003..0021aa527a3 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -244,7 +244,7 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container if (resultsReceiverEndpoint != null) { Trace.Info($"Queueing results file ({filePath}) for attachment upload ({attachmentName})"); - var stepId = context.Id; + var stepId = context.IsEmbedded ? context.EmbeddedId : context.Id; // Attachments must be added to the parent context (job), not the current context (step) context.Root.QueueSummaryFile(attachmentName, scrubbedFilePath, stepId); } From 4cb3cb29622ca3f79f40f6db80ba1b1780739e9a Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 28 Feb 2024 15:08:31 -0500 Subject: [PATCH 02/68] Bump runner version to match the latest patch release (#3175) --- src/runnerversion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runnerversion b/src/runnerversion index bba6eff1556..3a6182640d3 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.314.0 +2.314.1 From aa90563caecb8fcd2a1a84fc9c7833a56f6c40f4 Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Thu, 29 Feb 2024 10:39:29 -0500 Subject: [PATCH 03/68] don't crash listener on getting job exceptions (#3177) --- src/Runner.Listener/Runner.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index f44db4cb548..e38cd0570a8 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -567,6 +567,11 @@ await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, Trace.Info("Job is already acquired, skip this message."); continue; } + catch (Exception ex) + { + Trace.Error($"Caught exception from acquiring job message: {ex}"); + continue; + } } jobDispatcher.Run(jobRequestMessage, runOnce); From 86d6211c75d485afadfa4de34aed40a11d5458b5 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Mon, 4 Mar 2024 11:32:21 +0100 Subject: [PATCH 04/68] Remove -f flag in wait when manually trap signal (#3182) * Remove -f flag in wait when manually trap signal * Remove extra empty line --- src/Misc/layoutroot/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Misc/layoutroot/run.sh b/src/Misc/layoutroot/run.sh index 6b02ea18ff1..57f18ee00e1 100755 --- a/src/Misc/layoutroot/run.sh +++ b/src/Misc/layoutroot/run.sh @@ -38,7 +38,7 @@ runWithManualTrap() { cp -f "$DIR"/run-helper.sh.template "$DIR"/run-helper.sh "$DIR"/run-helper.sh $* & PID=$! - wait -f $PID + wait $PID returnCode=$? if [[ $returnCode -eq 2 ]]; then echo "Restarting runner..." @@ -84,4 +84,4 @@ if [[ -z "$RUNNER_MANUALLY_TRAP_SIG" ]]; then run $* else runWithManualTrap $* -fi \ No newline at end of file +fi From 2c8c9416224ad5e9926626d1360311568eefefd1 Mon Sep 17 00:00:00 2001 From: Patrick Carnahan Date: Tue, 5 Mar 2024 11:13:16 -0500 Subject: [PATCH 05/68] consume new pipelines service url in handlers (#3185) * consume pipelines service url if present updates how the `ACTIONS_RUNTIME_URL` variable is set to utilize a new value, `PipelinesServiceUrl` if present in the endpoint. if this value is not present then the existing system connection endpoint is used to retain backward compatibility. * consume pipelines url updates how the `ACTIONS_RUNTIME_URL` variable is set to utilize a new value, `PipelinesServiceUrl` if present in the endpoint. if this value is not present then the existing system connection endpoint is used to retain backward compatibility. --- src/Runner.Worker/Handlers/ContainerActionHandler.cs | 4 ++++ src/Runner.Worker/Handlers/NodeScriptActionHandler.cs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/Runner.Worker/Handlers/ContainerActionHandler.cs b/src/Runner.Worker/Handlers/ContainerActionHandler.cs index eb75bb59c14..775ce2f0428 100644 --- a/src/Runner.Worker/Handlers/ContainerActionHandler.cs +++ b/src/Runner.Worker/Handlers/ContainerActionHandler.cs @@ -223,6 +223,10 @@ public async Task RunAsync(ActionRunStage stage) { Environment["ACTIONS_CACHE_URL"] = cacheUrl; } + if (systemConnection.Data.TryGetValue("PipelinesServiceUrl", out var pipelinesServiceUrl) && !string.IsNullOrEmpty(pipelinesServiceUrl)) + { + Environment["ACTIONS_RUNTIME_URL"] = pipelinesServiceUrl; + } if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) && !string.IsNullOrEmpty(generateIdTokenUrl)) { Environment["ACTIONS_ID_TOKEN_REQUEST_URL"] = generateIdTokenUrl; diff --git a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs index 32d4eb08483..3426631f60a 100644 --- a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs +++ b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs @@ -58,6 +58,10 @@ public async Task RunAsync(ActionRunStage stage) { Environment["ACTIONS_CACHE_URL"] = cacheUrl; } + if (systemConnection.Data.TryGetValue("PipelinesServiceUrl", out var pipelinesServiceUrl) && !string.IsNullOrEmpty(pipelinesServiceUrl)) + { + Environment["ACTIONS_RUNTIME_URL"] = pipelinesServiceUrl; + } if (systemConnection.Data.TryGetValue("GenerateIdTokenUrl", out var generateIdTokenUrl) && !string.IsNullOrEmpty(generateIdTokenUrl)) { Environment["ACTIONS_ID_TOKEN_REQUEST_URL"] = generateIdTokenUrl; From 692d910868f35df3de35617fc5cd7c036dd2dad1 Mon Sep 17 00:00:00 2001 From: Tatyana Kostromskaya <32135588+takost@users.noreply.github.com> Date: Thu, 14 Mar 2024 14:12:08 +0100 Subject: [PATCH 06/68] Add ability to enforce actions to run on node20 (#3192) Add options to enforce actions execute on node20 --- src/Runner.Common/Constants.cs | 5 +++ src/Runner.Worker/Handlers/HandlerFactory.cs | 39 +++++++++++++++++++ .../Handlers/NodeScriptActionHandler.cs | 5 +++ src/Runner.Worker/JobRunner.cs | 12 ++++++ 4 files changed, 61 insertions(+) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 177e3c98f3f..98d9871c835 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -180,6 +180,9 @@ public static class Features public static readonly string DeprecatedNodeVersion = "node16"; public static readonly string EnforcedNode12DetectedAfterEndOfLife = "The following actions uses node12 which is deprecated and will be forced to run on node16: {0}. For more info: https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/"; public static readonly string EnforcedNode12DetectedAfterEndOfLifeEnvVariable = "Node16ForceActionsWarnings"; + public static readonly string EnforcedNode16DetectedAfterEndOfLife = "The following actions uses Node.js version which is deprecated and will be forced to run on node20: {0}. For more info: https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/"; + public static readonly string EnforcedNode16DetectedAfterEndOfLifeEnvVariable = "Node20ForceActionsWarnings"; + } public static class RunnerEvent @@ -251,6 +254,7 @@ public static class Actions public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG"; public static readonly string StepDebug = "ACTIONS_STEP_DEBUG"; public static readonly string AllowActionsUseUnsecureNodeVersion = "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION"; + public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20"; } public static class Agent @@ -262,6 +266,7 @@ public static class Agent public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION"; public static readonly string PrintLogToStdout = "ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT"; public static readonly string ActionArchiveCacheDirectory = "ACTIONS_RUNNER_ACTION_ARCHIVE_CACHE"; + public static readonly string ManualForceActionsToNode20 = "FORCE_JAVASCRIPT_ACTIONS_TO_NODE20"; } public static class System diff --git a/src/Runner.Worker/Handlers/HandlerFactory.cs b/src/Runner.Worker/Handlers/HandlerFactory.cs index 5f1fce0cf35..f857f89a9da 100644 --- a/src/Runner.Worker/Handlers/HandlerFactory.cs +++ b/src/Runner.Worker/Handlers/HandlerFactory.cs @@ -84,6 +84,45 @@ public IHandler Create( } nodeData.NodeVersion = "node16"; } + + var localForceActionsToNode20 = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Agent.ManualForceActionsToNode20)); + executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Variables.Actions.ManualForceActionsToNode20, out var workflowForceActionsToNode20); + var enforceNode20Locally = !string.IsNullOrWhiteSpace(workflowForceActionsToNode20) ? StringUtil.ConvertToBoolean(workflowForceActionsToNode20) : localForceActionsToNode20; + if (string.Equals(nodeData.NodeVersion, "node16") + && ((executionContext.Global.Variables.GetBoolean("DistributedTask.ForceGithubJavascriptActionsToNode20") ?? false) || enforceNode20Locally)) + { + executionContext.Global.EnvironmentVariables.TryGetValue(Constants.Variables.Actions.AllowActionsUseUnsecureNodeVersion, out var workflowOptOut); + var isWorkflowOptOutSet = !string.IsNullOrWhiteSpace(workflowOptOut); + var isLocalOptOut = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Actions.AllowActionsUseUnsecureNodeVersion)); + bool isOptOut = isWorkflowOptOutSet ? StringUtil.ConvertToBoolean(workflowOptOut) : isLocalOptOut; + + if (!isOptOut) + { + var repoAction = action as Pipelines.RepositoryPathReference; + if (repoAction != null) + { + var warningActions = new HashSet(); + if (executionContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings)) + { + warningActions = StringUtil.ConvertFromJson>(node20ForceWarnings); + } + + string repoActionFullName; + if (string.IsNullOrEmpty(repoAction.Name)) + { + repoActionFullName = repoAction.Path; // local actions don't have a 'Name' + } + else + { + repoActionFullName = $"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}"; + } + + warningActions.Add(repoActionFullName); + executionContext.Global.Variables.Set(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, StringUtil.ConvertToJson(warningActions)); + } + nodeData.NodeVersion = "node20"; + } + } (handler as INodeScriptActionHandler).Data = nodeData; } else if (data.ExecutionType == ActionExecutionType.Script) diff --git a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs index 3426631f60a..9d412a72314 100644 --- a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs +++ b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs @@ -118,6 +118,11 @@ public async Task RunAsync(ActionRunStage stage) { Data.NodeVersion = "node16"; } + + if (forcedNodeVersion == "node20" && Data.NodeVersion != "node20") + { + Data.NodeVersion = "node20"; + } var nodeRuntimeVersion = await StepHost.DetermineNodeRuntimeVersion(ExecutionContext, Data.NodeVersion); string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeRuntimeVersion, "bin", $"node{IOUtil.ExeExtension}"); diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 6db477214da..69ebed556c2 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -298,6 +298,12 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions)); } + if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings) && (jobContext.Global.Variables.GetBoolean("DistributedTask.ForceGithubJavascriptActionsToNode20") ?? false)) + { + var actions = string.Join(", ", StringUtil.ConvertFromJson>(node20ForceWarnings)); + jobContext.Warning(string.Format(Constants.Runner.EnforcedNode16DetectedAfterEndOfLife, actions)); + } + await ShutdownQueue(throwOnFailure: false); // Make sure to clean temp after file upload since they may be pending fileupload still use the TEMP dir. @@ -405,6 +411,12 @@ private async Task CompleteJobAsync(IJobServer jobServer, IExecution jobContext.Warning(string.Format(Constants.Runner.EnforcedNode12DetectedAfterEndOfLife, actions)); } + if (jobContext.Global.Variables.TryGetValue(Constants.Runner.EnforcedNode16DetectedAfterEndOfLifeEnvVariable, out var node20ForceWarnings)) + { + var actions = string.Join(", ", StringUtil.ConvertFromJson>(node20ForceWarnings)); + jobContext.Warning(string.Format(Constants.Runner.EnforcedNode16DetectedAfterEndOfLife, actions)); + } + try { var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: true); From 93bc1cd91859f054db0a833c76f4f8d492c7a950 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Fri, 15 Mar 2024 16:13:29 +0100 Subject: [PATCH 07/68] Bump hook version to 0.6.0 (#3203) --- images/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/Dockerfile b/images/Dockerfile index f95a5e47e93..cfde7020bcd 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -4,7 +4,7 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy as build ARG TARGETOS ARG TARGETARCH ARG RUNNER_VERSION -ARG RUNNER_CONTAINER_HOOKS_VERSION=0.5.1 +ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.0 ARG DOCKER_VERSION=25.0.2 ARG BUILDX_VERSION=0.12.1 From 82e01c61731050ae75048848440e9cca02a919a4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 17 Mar 2024 22:42:44 -0400 Subject: [PATCH 08/68] Upgrade dotnet sdk to v6.0.420 (#3211) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .devcontainer/devcontainer.json | 2 +- src/dev.sh | 2 +- src/global.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index eecb81148e6..a05670d7a86 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,7 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:1": {}, "ghcr.io/devcontainers/features/dotnet": { - "version": "6.0.419" + "version": "6.0.420" }, "ghcr.io/devcontainers/features/node:1": { "version": "16" diff --git a/src/dev.sh b/src/dev.sh index 485ca520ee3..643ab559a7f 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -17,7 +17,7 @@ LAYOUT_DIR="$SCRIPT_DIR/../_layout" DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x" PACKAGE_DIR="$SCRIPT_DIR/../_package" DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk" -DOTNETSDK_VERSION="6.0.419" +DOTNETSDK_VERSION="6.0.420" DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION" RUNNER_VERSION=$(cat runnerversion) diff --git a/src/global.json b/src/global.json index bf923c69ce8..f35fb2f37f1 100644 --- a/src/global.json +++ b/src/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "6.0.419" + "version": "6.0.420" } } From bc8b6e01528a1232341773be73078f7639f17187 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Wed, 20 Mar 2024 16:16:41 +0100 Subject: [PATCH 09/68] Bump docker version and docker buildx version (#3208) --- images/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/Dockerfile b/images/Dockerfile index cfde7020bcd..2bd1a338473 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -5,8 +5,8 @@ ARG TARGETOS ARG TARGETARCH ARG RUNNER_VERSION ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.0 -ARG DOCKER_VERSION=25.0.2 -ARG BUILDX_VERSION=0.12.1 +ARG DOCKER_VERSION=25.0.4 +ARG BUILDX_VERSION=0.13.1 RUN apt update -y && apt install curl unzip -y From 4b8514566152de72a70f922d61c2e4fa10d39121 Mon Sep 17 00:00:00 2001 From: Jacob Wallraff Date: Thu, 21 Mar 2024 11:50:45 -0700 Subject: [PATCH 10/68] Handle new non-retryable exception type (#3191) * Handle new non-retryable exception type * Update ActionManager.cs --- src/Runner.Worker/ActionManager.cs | 3 ++- src/Sdk/DTWebApi/WebApi/Exceptions.cs | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index bf7838c1b5a..0dd891c1765 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -703,11 +703,12 @@ private async Task BuildActionContainerAsync(IExecutionContext executionContext, catch (Exception ex) when (!executionContext.CancellationToken.IsCancellationRequested) // Do not retry if the run is cancelled. { // UnresolvableActionDownloadInfoException is a 422 client error, don't retry + // NonRetryableActionDownloadInfoException is an non-retryable exception from Actions // Some possible cases are: // * Repo is rate limited // * Repo or tag doesn't exist, or isn't public // * Policy validation failed - if (attempt < 3 && !(ex is WebApi.UnresolvableActionDownloadInfoException)) + if (attempt < 3 && !(ex is WebApi.UnresolvableActionDownloadInfoException) && !(ex is WebApi.NonRetryableActionDownloadInfoException)) { executionContext.Output($"Failed to resolve action download info. Error: {ex.Message}"); executionContext.Debug(ex.ToString()); diff --git a/src/Sdk/DTWebApi/WebApi/Exceptions.cs b/src/Sdk/DTWebApi/WebApi/Exceptions.cs index 97505bb6a41..536bf755055 100644 --- a/src/Sdk/DTWebApi/WebApi/Exceptions.cs +++ b/src/Sdk/DTWebApi/WebApi/Exceptions.cs @@ -2498,6 +2498,25 @@ protected UnresolvableActionDownloadInfoException(SerializationInfo info, Stream } } + [Serializable] + public class NonRetryableActionDownloadInfoException : DistributedTaskException + { + public NonRetryableActionDownloadInfoException(String message) + : base(message) + { + } + + public NonRetryableActionDownloadInfoException(String message, Exception innerException) + : base(message, innerException) + { + } + + protected NonRetryableActionDownloadInfoException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } + [Serializable] public sealed class FailedToResolveActionDownloadInfoException : DistributedTaskException { From 8ebf298bcd70548f5c1ce4c22b767a91339293db Mon Sep 17 00:00:00 2001 From: Luke Tomlinson Date: Thu, 21 Mar 2024 16:30:34 -0400 Subject: [PATCH 11/68] Always Delete Actions Service Session (#3214) * Delete Actions Service session always * update tes --- src/Runner.Listener/MessageListener.cs | 4 ++-- src/Test/L0/Listener/MessageListenerL0.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 7be8b0bce84..ab91cff0508 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -188,12 +188,12 @@ public async Task DeleteSessionAsync() { using (var ts = new CancellationTokenSource(TimeSpan.FromSeconds(30))) { + await _runnerServer.DeleteAgentSessionAsync(_settings.PoolId, _session.SessionId, ts.Token); + if (_isBrokerSession) { await _brokerServer.DeleteSessionAsync(ts.Token); - return; } - await _runnerServer.DeleteAgentSessionAsync(_settings.PoolId, _session.SessionId, ts.Token); } } else diff --git a/src/Test/L0/Listener/MessageListenerL0.cs b/src/Test/L0/Listener/MessageListenerL0.cs index 57a1f60d800..b6837ad9a11 100644 --- a/src/Test/L0/Listener/MessageListenerL0.cs +++ b/src/Test/L0/Listener/MessageListenerL0.cs @@ -272,7 +272,7 @@ public async void DeleteSessionWithBrokerMigration() //Assert _runnerServer .Verify(x => x.DeleteAgentSessionAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny()), Times.Never()); + _settings.PoolId, expectedBrokerSession.SessionId, It.IsAny()), Times.Once()); _brokerServer .Verify(x => x.DeleteSessionAsync(It.IsAny()), Times.Once()); } From a52c53955c0c43d51654de2779a05c5bd1091c0f Mon Sep 17 00:00:00 2001 From: Luke Tomlinson Date: Fri, 22 Mar 2024 11:38:53 -0400 Subject: [PATCH 12/68] Prepare v2.315.0 release (#3216) --- releaseNote.md | 27 +++++++++++++++------------ src/runnerversion | 2 +- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index e9f86ca4f37..cf0c0688ca5 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,18 +1,21 @@ ## What's Changed -* Prepare v2.313.0 Release by @luketomlinson in https://github.com/actions/runner/pull/3137 -* Pass RunnerOS during job acquire. by @TingluoHuang in https://github.com/actions/runner/pull/3140 -* Process `snapshot` tokens by @davidomid in https://github.com/actions/runner/pull/3135 -* Update dotnet sdk to latest version @6.0.419 by @github-actions in https://github.com/actions/runner/pull/3158 -* handle broker run service exception handling by @yaananth in https://github.com/actions/runner/pull/3163 -* Add a retry logic to docker login operation by @enescakir in https://github.com/actions/runner/pull/3089 -* Broker fixes for token refreshes and AccessDeniedException by @luketomlinson in https://github.com/actions/runner/pull/3161 -* Remove USE_BROKER_FLOW by @luketomlinson in https://github.com/actions/runner/pull/3162 -* Refresh Token for BrokerServer by @luketomlinson in https://github.com/actions/runner/pull/3167 -* Better step timeout message. by @TingluoHuang in https://github.com/actions/runner/pull/3166 +* fix summaries for actions results by @SrRyan in https://github.com/actions/runner/pull/3174 +* Bump runner version to match the latest patch release by @TingluoHuang in https://github.com/actions/runner/pull/3175 +* don't crash listener on getting job exceptions for run-service by @yaananth in https://github.com/actions/runner/pull/3177 +* Remove -f flag in wait when manually trap signal by @nikola-jokic in https://github.com/actions/runner/pull/3182 +* consume new pipelines service url in handlers by @patrickcarnahan in https://github.com/actions/runner/pull/3185 +* Add ability to enforce actions to run on node20 by @takost in https://github.com/actions/runner/pull/3192 +* Bump hook version to 0.6.0 by @nikola-jokic in https://github.com/actions/runner/pull/3203 +* Update dotnet sdk to latest version @6.0.420 by @github-actions in https://github.com/actions/runner/pull/3211 +* Bump docker version and docker buildx version by @nikola-jokic in https://github.com/actions/runner/pull/3208 +* Handle new non-retryable exception type by @thyeggman in https://github.com/actions/runner/pull/3191 +* Always Delete Actions Service Session by @luketomlinson in https://github.com/actions/runner/pull/3214 ## New Contributors -* @davidomid made their first contribution in https://github.com/actions/runner/pull/3135 -* @enescakir made their first contribution in https://github.com/actions/runner/pull/3089 +* @SrRyan made their first contribution in https://github.com/actions/runner/pull/3174 +* @patrickcarnahan made their first contribution in https://github.com/actions/runner/pull/3185 + +**Full Changelog**: https://github.com/actions/runner/compare/v2.314.1...v2.315.0 **Full Changelog**: https://github.com/actions/runner/compare/v2.313.0...v2.314.0 diff --git a/src/runnerversion b/src/runnerversion index 3a6182640d3..bb8021df34e 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.314.1 +2.315.0 From 77e0bfbb8a8fde1f01fc1cf1ed2d7f0e81a0a407 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Sat, 23 Mar 2024 23:12:39 -0400 Subject: [PATCH 13/68] Load '_runnerSettings' in the early point of JobRunner.cs (#3218) --- src/Runner.Worker/JobRunner.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 69ebed556c2..7edff9ac363 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -42,6 +42,7 @@ public async Task RunAsync(AgentJobRequestMessage message, Cancellat Trace.Info("Job ID {0}", message.JobId); DateTime jobStartTimeUtc = DateTime.UtcNow; + _runnerSettings = HostContext.GetService().GetSettings(); IRunnerService server = null; // add orchestration id to useragent for better correlation. @@ -164,8 +165,6 @@ public async Task RunAsync(AgentJobRequestMessage message, Cancellat jobContext.SetRunnerContext("os", VarUtil.OS); jobContext.SetRunnerContext("arch", VarUtil.OSArchitecture); - - _runnerSettings = HostContext.GetService().GetSettings(); jobContext.SetRunnerContext("name", _runnerSettings.AgentName); if (jobContext.Global.Variables.TryGetValue(WellKnownDistributedTaskVariables.RunnerEnvironment, out var runnerEnvironment)) From f467e9e1255530d3bf2e33f580d041925ab01951 Mon Sep 17 00:00:00 2001 From: eeSquared Date: Wed, 27 Mar 2024 14:49:58 -0400 Subject: [PATCH 14/68] Add new SessionConflict return code (#3215) * Add new SessionConflict return code * formatting * Change return type of CreateSessionAsync to new enum * Update entry scripts to handle new exit code * Move enum --- src/Misc/layoutbin/RunnerService.js | 5 +++ src/Misc/layoutroot/run-helper.cmd.template | 5 +++ src/Misc/layoutroot/run-helper.sh.template | 3 ++ src/Runner.Common/Constants.cs | 1 + src/Runner.Listener/BrokerMessageListener.cs | 14 +++++--- src/Runner.Listener/MessageListener.cs | 25 +++++++++++---- src/Runner.Listener/Runner.cs | 7 +++- .../L0/Listener/BrokerMessageListenerL0.cs | 4 +-- src/Test/L0/Listener/MessageListenerL0.cs | 32 +++++++++---------- src/Test/L0/Listener/RunnerL0.cs | 12 +++---- 10 files changed, 71 insertions(+), 37 deletions(-) diff --git a/src/Misc/layoutbin/RunnerService.js b/src/Misc/layoutbin/RunnerService.js index ba0a8c659d7..1024e8a5e77 100644 --- a/src/Misc/layoutbin/RunnerService.js +++ b/src/Misc/layoutbin/RunnerService.js @@ -114,6 +114,11 @@ var runService = function () { ); stopping = true; } + } else if (code === 5) { + console.log( + "Runner listener exit with Session Conflict error, stop the service, no retry needed." + ); + stopping = true; } else { var messagePrefix = "Runner listener exit with undefined return code"; unknownFailureRetryCount++; diff --git a/src/Misc/layoutroot/run-helper.cmd.template b/src/Misc/layoutroot/run-helper.cmd.template index 221e8b1c024..6b594d4f357 100644 --- a/src/Misc/layoutroot/run-helper.cmd.template +++ b/src/Misc/layoutroot/run-helper.cmd.template @@ -49,5 +49,10 @@ if %ERRORLEVEL% EQU 4 ( exit /b 1 ) +if %ERRORLEVEL% EQU 5 ( + echo "Runner listener exit with Session Conflict error, stop the service, no retry needed." + exit /b 0 +) + echo "Exiting after unknown error code: %ERRORLEVEL%" exit /b 0 \ No newline at end of file diff --git a/src/Misc/layoutroot/run-helper.sh.template b/src/Misc/layoutroot/run-helper.sh.template index 743fd8b6959..9f2b3cc4457 100755 --- a/src/Misc/layoutroot/run-helper.sh.template +++ b/src/Misc/layoutroot/run-helper.sh.template @@ -70,6 +70,9 @@ elif [[ $returnCode == 4 ]]; then "$DIR"/safe_sleep.sh 1 done exit 2 +elif [[ $returnCode == 5 ]]; then + echo "Runner listener exit with Session Conflict error, stop the service, no retry needed." + exit 0 else echo "Exiting with unknown error code: ${returnCode}" exit 0 diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 98d9871c835..9378104c435 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -153,6 +153,7 @@ public static class ReturnCode public const int RetryableError = 2; public const int RunnerUpdating = 3; public const int RunOnceRunnerUpdating = 4; + public const int SessionConflict = 5; } public static class Features diff --git a/src/Runner.Listener/BrokerMessageListener.cs b/src/Runner.Listener/BrokerMessageListener.cs index 6767d0beb39..8f44fe84372 100644 --- a/src/Runner.Listener/BrokerMessageListener.cs +++ b/src/Runner.Listener/BrokerMessageListener.cs @@ -42,7 +42,7 @@ public override void Initialize(IHostContext hostContext) _brokerServer = HostContext.GetService(); } - public async Task CreateSessionAsync(CancellationToken token) + public async Task CreateSessionAsync(CancellationToken token) { Trace.Entering(); @@ -99,7 +99,7 @@ public async Task CreateSessionAsync(CancellationToken token) encounteringError = false; } - return true; + return CreateSessionResult.Success; } catch (OperationCanceledException) when (token.IsCancellationRequested) { @@ -123,7 +123,7 @@ public async Task CreateSessionAsync(CancellationToken token) if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase)) { _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently."); - return false; + return CreateSessionResult.Failure; } // Check whether we get 401 because the runner registration already removed by the service. @@ -134,14 +134,18 @@ public async Task CreateSessionAsync(CancellationToken token) if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase)) { _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently."); - return false; + return CreateSessionResult.Failure; } } if (!IsSessionCreationExceptionRetriable(ex)) { _term.WriteError($"Failed to create session. {ex.Message}"); - return false; + if (ex is TaskAgentSessionConflictException) + { + return CreateSessionResult.SessionConflict; + } + return CreateSessionResult.Failure; } if (!encounteringError) //print the message only on the first error diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index ab91cff0508..403c2bfc720 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -18,10 +18,17 @@ namespace GitHub.Runner.Listener { + public enum CreateSessionResult + { + Success, + Failure, + SessionConflict + } + [ServiceLocator(Default = typeof(MessageListener))] public interface IMessageListener : IRunnerService { - Task CreateSessionAsync(CancellationToken token); + Task CreateSessionAsync(CancellationToken token); Task DeleteSessionAsync(); Task GetNextMessageAsync(CancellationToken token); Task DeleteMessageAsync(TaskAgentMessage message); @@ -59,7 +66,7 @@ public override void Initialize(IHostContext hostContext) _brokerServer = hostContext.GetService(); } - public async Task CreateSessionAsync(CancellationToken token) + public async Task CreateSessionAsync(CancellationToken token) { Trace.Entering(); @@ -123,7 +130,7 @@ public async Task CreateSessionAsync(CancellationToken token) encounteringError = false; } - return true; + return CreateSessionResult.Success; } catch (OperationCanceledException) when (token.IsCancellationRequested) { @@ -147,7 +154,7 @@ public async Task CreateSessionAsync(CancellationToken token) if (string.Equals(vssOAuthEx.Error, "invalid_client", StringComparison.OrdinalIgnoreCase)) { _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently."); - return false; + return CreateSessionResult.Failure; } // Check whether we get 401 because the runner registration already removed by the service. @@ -158,14 +165,18 @@ public async Task CreateSessionAsync(CancellationToken token) if (string.Equals(authError, "invalid_client", StringComparison.OrdinalIgnoreCase)) { _term.WriteError("Failed to create a session. The runner registration has been deleted from the server, please re-configure. Runner registrations are automatically deleted for runners that have not connected to the service recently."); - return false; + return CreateSessionResult.Failure; } } if (!IsSessionCreationExceptionRetriable(ex)) { _term.WriteError($"Failed to create session. {ex.Message}"); - return false; + if (ex is TaskAgentSessionConflictException) + { + return CreateSessionResult.SessionConflict; + } + return CreateSessionResult.Failure; } if (!encounteringError) //print the message only on the first error @@ -303,7 +314,7 @@ public async Task GetNextMessageAsync(CancellationToken token) Trace.Error(ex); // don't retry if SkipSessionRecover = true, DT service will delete agent session to stop agent from taking more jobs. - if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && await CreateSessionAsync(token)) + if (ex is TaskAgentSessionExpiredException && !_settings.SkipSessionRecover && (await CreateSessionAsync(token) == CreateSessionResult.Success)) { Trace.Info($"{nameof(TaskAgentSessionExpiredException)} received, recovered by recreate session."); } diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index e38cd0570a8..9acad839546 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -359,7 +359,12 @@ private async Task RunAsync(RunnerSettings settings, bool runOnce = false) { Trace.Info(nameof(RunAsync)); _listener = GetMesageListener(settings); - if (!await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken)) + CreateSessionResult createSessionResult = await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken); + if (createSessionResult == CreateSessionResult.SessionConflict) + { + return Constants.Runner.ReturnCode.SessionConflict; + } + else if (createSessionResult == CreateSessionResult.Failure) { return Constants.Runner.ReturnCode.TerminatedError; } diff --git a/src/Test/L0/Listener/BrokerMessageListenerL0.cs b/src/Test/L0/Listener/BrokerMessageListenerL0.cs index 7dface3b2eb..64a71515c16 100644 --- a/src/Test/L0/Listener/BrokerMessageListenerL0.cs +++ b/src/Test/L0/Listener/BrokerMessageListenerL0.cs @@ -56,11 +56,11 @@ public async void CreatesSession() BrokerMessageListener listener = new(); listener.Initialize(tc); - bool result = await listener.CreateSessionAsync(tokenSource.Token); + CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token); trace.Info("result: {0}", result); // Assert. - Assert.True(result); + Assert.Equal(CreateSessionResult.Success, result); _brokerServer .Verify(x => x.CreateSessionAsync( It.Is(y => y != null), diff --git a/src/Test/L0/Listener/MessageListenerL0.cs b/src/Test/L0/Listener/MessageListenerL0.cs index b6837ad9a11..f44d4988928 100644 --- a/src/Test/L0/Listener/MessageListenerL0.cs +++ b/src/Test/L0/Listener/MessageListenerL0.cs @@ -75,11 +75,11 @@ public async void CreatesSession() MessageListener listener = new(); listener.Initialize(tc); - bool result = await listener.CreateSessionAsync(tokenSource.Token); + CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token); trace.Info("result: {0}", result); // Assert. - Assert.True(result); + Assert.Equal(CreateSessionResult.Success, result); _runnerServer .Verify(x => x.CreateAgentSessionAsync( _settings.PoolId, @@ -135,11 +135,11 @@ public async void CreatesSessionWithBrokerMigration() MessageListener listener = new(); listener.Initialize(tc); - bool result = await listener.CreateSessionAsync(tokenSource.Token); + CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token); trace.Info("result: {0}", result); // Assert. - Assert.True(result); + Assert.Equal(CreateSessionResult.Success, result); _runnerServer .Verify(x => x.CreateAgentSessionAsync( @@ -185,8 +185,8 @@ public async void DeleteSession() MessageListener listener = new(); listener.Initialize(tc); - bool result = await listener.CreateSessionAsync(tokenSource.Token); - Assert.True(result); + CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token); + Assert.Equal(CreateSessionResult.Success, result); _runnerServer .Setup(x => x.DeleteAgentSessionAsync( @@ -245,10 +245,10 @@ public async void DeleteSessionWithBrokerMigration() MessageListener listener = new(); listener.Initialize(tc); - bool result = await listener.CreateSessionAsync(tokenSource.Token); + CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token); trace.Info("result: {0}", result); - Assert.True(result); + Assert.Equal(CreateSessionResult.Success, result); _runnerServer .Verify(x => x.CreateAgentSessionAsync( @@ -309,8 +309,8 @@ public async void GetNextMessage() MessageListener listener = new(); listener.Initialize(tc); - bool result = await listener.CreateSessionAsync(tokenSource.Token); - Assert.True(result); + CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token); + Assert.Equal(CreateSessionResult.Success, result); var arMessages = new TaskAgentMessage[] { @@ -390,8 +390,8 @@ public async void GetNextMessageWithBrokerMigration() MessageListener listener = new(); listener.Initialize(tc); - bool result = await listener.CreateSessionAsync(tokenSource.Token); - Assert.True(result); + CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token); + Assert.Equal(CreateSessionResult.Success, result); var brokerMigrationMesage = new BrokerMigrationMessage(new Uri("https://actions.broker.com")); @@ -497,11 +497,11 @@ public async void CreateSessionWithOriginalCredential() MessageListener listener = new(); listener.Initialize(tc); - bool result = await listener.CreateSessionAsync(tokenSource.Token); + CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token); trace.Info("result: {0}", result); // Assert. - Assert.True(result); + Assert.Equal(CreateSessionResult.Success, result); _runnerServer .Verify(x => x.CreateAgentSessionAsync( _settings.PoolId, @@ -541,8 +541,8 @@ public async void SkipDeleteSession_WhenGetNextMessageGetTaskAgentAccessTokenExp MessageListener listener = new(); listener.Initialize(tc); - bool result = await listener.CreateSessionAsync(tokenSource.Token); - Assert.True(result); + CreateSessionResult result = await listener.CreateSessionAsync(tokenSource.Token); + Assert.Equal(CreateSessionResult.Success, result); _runnerServer .Setup(x => x.GetAgentMessageAsync( diff --git a/src/Test/L0/Listener/RunnerL0.cs b/src/Test/L0/Listener/RunnerL0.cs index 9c57f2adc36..10b6d8c0fed 100644 --- a/src/Test/L0/Listener/RunnerL0.cs +++ b/src/Test/L0/Listener/RunnerL0.cs @@ -88,7 +88,7 @@ public async void TestRunAsync() _configurationManager.Setup(x => x.IsConfigured()) .Returns(true); _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny())) - .Returns(Task.FromResult(true)); + .Returns(Task.FromResult(CreateSessionResult.Success)); _messageListener.Setup(x => x.GetNextMessageAsync(It.IsAny())) .Returns(async () => { @@ -184,7 +184,7 @@ public async void TestExecuteCommandForRunAsService(string[] args, bool configur _configStore.Setup(x => x.IsServiceConfigured()).Returns(configureAsService); _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny())) - .Returns(Task.FromResult(false)); + .Returns(Task.FromResult(CreateSessionResult.Failure)); var runner = new Runner.Listener.Runner(); runner.Initialize(hc); @@ -217,7 +217,7 @@ public async void TestMachineProvisionerCLI() .Returns(false); _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny())) - .Returns(Task.FromResult(false)); + .Returns(Task.FromResult(CreateSessionResult.Failure)); var runner = new Runner.Listener.Runner(); runner.Initialize(hc); @@ -263,7 +263,7 @@ public async void TestRunOnce() _configurationManager.Setup(x => x.IsConfigured()) .Returns(true); _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny())) - .Returns(Task.FromResult(true)); + .Returns(Task.FromResult(CreateSessionResult.Success)); _messageListener.Setup(x => x.GetNextMessageAsync(It.IsAny())) .Returns(async () => { @@ -363,7 +363,7 @@ public async void TestRunOnceOnlyTakeOneJobMessage() _configurationManager.Setup(x => x.IsConfigured()) .Returns(true); _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny())) - .Returns(Task.FromResult(true)); + .Returns(Task.FromResult(CreateSessionResult.Success)); _messageListener.Setup(x => x.GetNextMessageAsync(It.IsAny())) .Returns(async () => { @@ -458,7 +458,7 @@ public async void TestRunOnceHandleUpdateMessage() _configurationManager.Setup(x => x.IsConfigured()) .Returns(true); _messageListener.Setup(x => x.CreateSessionAsync(It.IsAny())) - .Returns(Task.FromResult(true)); + .Returns(Task.FromResult(CreateSessionResult.Success)); _messageListener.Setup(x => x.GetNextMessageAsync(It.IsAny())) .Returns(async () => { From 4a1e38095b53ebdd9eed320756794de2fd8d1a0e Mon Sep 17 00:00:00 2001 From: Aiqiao Yan <55104035+aiqiaoy@users.noreply.github.com> Date: Tue, 9 Apr 2024 14:13:07 -0400 Subject: [PATCH 15/68] =?UTF-8?q?backoff=20if=20we=20retried=20polling=20f?= =?UTF-8?q?or=20more=20than=2050=20times=20in=20less=20than=2030m=E2=80=A6?= =?UTF-8?q?=20(#3232)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * backoff if we retried polling for more than 50 times in less than 30minutes * run dotnet format * move delay to after no message trace * run dotnet format --- src/Runner.Listener/MessageListener.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 403c2bfc720..7b51423e6f5 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -236,6 +236,7 @@ public async Task GetNextMessageAsync(CancellationToken token) ArgUtil.NotNull(_settings, nameof(_settings)); bool encounteringError = false; int continuousError = 0; + int continuousEmptyMessage = 0; string errorMessage = string.Empty; Stopwatch heartbeat = new(); heartbeat.Restart(); @@ -359,16 +360,27 @@ public async Task GetNextMessageAsync(CancellationToken token) if (message == null) { + continuousEmptyMessage++; if (heartbeat.Elapsed > TimeSpan.FromMinutes(30)) { Trace.Info($"No message retrieved from session '{_session.SessionId}' within last 30 minutes."); heartbeat.Restart(); + continuousEmptyMessage = 0; } else { Trace.Verbose($"No message retrieved from session '{_session.SessionId}'."); } + if (continuousEmptyMessage > 50) + { + // retried more than 50 times in less than 30mins and still getting empty message + // something is not right on the service side, backoff for 15-30s before retry + _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval); + Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds); + await HostContext.Delay(_getNextMessageRetryInterval, token); + } + continue; } From 2d83e1d88fc9352f186d3d73cb8b9e67b3faf360 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 14 Apr 2024 22:47:26 -0400 Subject: [PATCH 16/68] Upgrade dotnet sdk to v6.0.421 (#3244) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .devcontainer/devcontainer.json | 2 +- src/dev.sh | 2 +- src/global.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a05670d7a86..032387bec99 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,7 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:1": {}, "ghcr.io/devcontainers/features/dotnet": { - "version": "6.0.420" + "version": "6.0.421" }, "ghcr.io/devcontainers/features/node:1": { "version": "16" diff --git a/src/dev.sh b/src/dev.sh index 643ab559a7f..0fc58761e53 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -17,7 +17,7 @@ LAYOUT_DIR="$SCRIPT_DIR/../_layout" DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x" PACKAGE_DIR="$SCRIPT_DIR/../_package" DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk" -DOTNETSDK_VERSION="6.0.420" +DOTNETSDK_VERSION="6.0.421" DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION" RUNNER_VERSION=$(cat runnerversion) diff --git a/src/global.json b/src/global.json index f35fb2f37f1..e7028fe0dd4 100644 --- a/src/global.json +++ b/src/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "6.0.420" + "version": "6.0.421" } } From 2e0eb2c11f7c7216492f4496c1e7cc216ae26ce4 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 16 Apr 2024 16:52:10 -0400 Subject: [PATCH 17/68] Cleanup enabled feature flags. (#3246) * Cleanup FF 'DistributedTask.UseWhich2'. * more ff. --- src/Runner.Common/JobServerQueue.cs | 11 +- src/Runner.Sdk/Util/WhichUtil.cs | 123 ------------ src/Runner.Worker/ActionManager.cs | 66 +++---- src/Runner.Worker/Handlers/ScriptHandler.cs | 83 +------- src/Runner.Worker/JobRunner.cs | 11 +- src/Test/L0/Util/WhichUtilL0.cs | 205 -------------------- src/Test/L0/Worker/ActionManagerL0.cs | 2 - 7 files changed, 48 insertions(+), 453 deletions(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index c1425b80721..9c27bf51d0e 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -19,7 +19,7 @@ public interface IJobServerQueue : IRunnerService, IThrottlingReporter TaskCompletionSource JobRecordUpdated { get; } event EventHandler JobServerQueueThrottling; Task ShutdownAsync(); - void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false); + void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false); void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null); void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource); void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines); @@ -104,11 +104,10 @@ public override void Initialize(IHostContext hostContext) _resultsServer = hostContext.GetService(); } - public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false, bool enableTelemetry = false) + public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServiceOnly = false) { Trace.Entering(); _resultsServiceOnly = resultsServiceOnly; - _enableTelemetry = enableTelemetry; var serviceEndPoint = jobRequest.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); @@ -139,6 +138,12 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServi _resultsClientInitiated = true; } + // Enable telemetry if we have both results service and actions service + if (_resultsClientInitiated && !_resultsServiceOnly) + { + _enableTelemetry = true; + } + if (_queueInProcess) { Trace.Info("No-opt, all queue process tasks are running."); diff --git a/src/Runner.Sdk/Util/WhichUtil.cs b/src/Runner.Sdk/Util/WhichUtil.cs index fde4fa2f6a9..ef7683a2d8f 100644 --- a/src/Runner.Sdk/Util/WhichUtil.cs +++ b/src/Runner.Sdk/Util/WhichUtil.cs @@ -7,129 +7,6 @@ namespace GitHub.Runner.Sdk public static class WhichUtil { public static string Which(string command, bool require = false, ITraceWriter trace = null, string prependPath = null) - { - ArgUtil.NotNullOrEmpty(command, nameof(command)); - trace?.Info($"Which: '{command}'"); - if (Path.IsPathFullyQualified(command) && File.Exists(command)) - { - trace?.Info($"Fully qualified path: '{command}'"); - return command; - } - string path = Environment.GetEnvironmentVariable(PathUtil.PathVariable); - if (string.IsNullOrEmpty(path)) - { - trace?.Info("PATH environment variable not defined."); - path = path ?? string.Empty; - } - if (!string.IsNullOrEmpty(prependPath)) - { - path = PathUtil.PrependPath(prependPath, path); - } - - string[] pathSegments = path.Split(new Char[] { Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries); - for (int i = 0; i < pathSegments.Length; i++) - { - pathSegments[i] = Environment.ExpandEnvironmentVariables(pathSegments[i]); - } - - foreach (string pathSegment in pathSegments) - { - if (!string.IsNullOrEmpty(pathSegment) && Directory.Exists(pathSegment)) - { - string[] matches = null; -#if OS_WINDOWS - string pathExt = Environment.GetEnvironmentVariable("PATHEXT"); - if (string.IsNullOrEmpty(pathExt)) - { - // XP's system default value for PATHEXT system variable - pathExt = ".com;.exe;.bat;.cmd;.vbs;.vbe;.js;.jse;.wsf;.wsh"; - } - - string[] pathExtSegments = pathExt.Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries); - - // if command already has an extension. - if (pathExtSegments.Any(ext => command.EndsWith(ext, StringComparison.OrdinalIgnoreCase))) - { - try - { - matches = Directory.GetFiles(pathSegment, command); - } - catch (UnauthorizedAccessException ex) - { - trace?.Info("Ignore UnauthorizedAccess exception during Which."); - trace?.Verbose(ex.ToString()); - } - - if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace)) - { - trace?.Info($"Location: '{matches.First()}'"); - return matches.First(); - } - } - else - { - string searchPattern; - searchPattern = StringUtil.Format($"{command}.*"); - try - { - matches = Directory.GetFiles(pathSegment, searchPattern); - } - catch (UnauthorizedAccessException ex) - { - trace?.Info("Ignore UnauthorizedAccess exception during Which."); - trace?.Verbose(ex.ToString()); - } - - if (matches != null && matches.Length > 0) - { - // add extension. - for (int i = 0; i < pathExtSegments.Length; i++) - { - string fullPath = Path.Combine(pathSegment, $"{command}{pathExtSegments[i]}"); - if (matches.Any(p => p.Equals(fullPath, StringComparison.OrdinalIgnoreCase)) && IsPathValid(fullPath, trace)) - { - trace?.Info($"Location: '{fullPath}'"); - return fullPath; - } - } - } - } -#else - try - { - matches = Directory.GetFiles(pathSegment, command); - } - catch (UnauthorizedAccessException ex) - { - trace?.Info("Ignore UnauthorizedAccess exception during Which."); - trace?.Verbose(ex.ToString()); - } - - if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace)) - { - trace?.Info($"Location: '{matches.First()}'"); - return matches.First(); - } -#endif - } - } - -#if OS_WINDOWS - trace?.Info($"{command}: command not found. Make sure '{command}' is installed and its location included in the 'Path' environment variable."); -#else - trace?.Info($"{command}: command not found. Make sure '{command}' is installed and its location included in the 'PATH' environment variable."); -#endif - if (require) - { - throw new FileNotFoundException( - message: $"{command}: command not found", - fileName: command); - } - - return null; - } - - public static string Which2(string command, bool require = false, ITraceWriter trace = null, string prependPath = null) { ArgUtil.NotNullOrEmpty(command, nameof(command)); trace?.Info($"Which2: '{command}'"); diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 0dd891c1765..227e7f4a5e6 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -797,43 +797,40 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont try { var useActionArchiveCache = false; - if (executionContext.Global.Variables.GetBoolean("DistributedTask.UseActionArchiveCache") == true) + var hasActionArchiveCache = false; + var actionArchiveCacheDir = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory); + if (!string.IsNullOrEmpty(actionArchiveCacheDir) && + Directory.Exists(actionArchiveCacheDir)) { - var hasActionArchiveCache = false; - var actionArchiveCacheDir = Environment.GetEnvironmentVariable(Constants.Variables.Agent.ActionArchiveCacheDirectory); - if (!string.IsNullOrEmpty(actionArchiveCacheDir) && - Directory.Exists(actionArchiveCacheDir)) - { - hasActionArchiveCache = true; - Trace.Info($"Check if action archive '{downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha}' already exists in cache directory '{actionArchiveCacheDir}'"); + hasActionArchiveCache = true; + Trace.Info($"Check if action archive '{downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha}' already exists in cache directory '{actionArchiveCacheDir}'"); #if OS_WINDOWS - var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.zip"); + var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.zip"); #else - var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.tar.gz"); + var cacheArchiveFile = Path.Combine(actionArchiveCacheDir, downloadInfo.ResolvedNameWithOwner.Replace(Path.DirectorySeparatorChar, '_').Replace(Path.AltDirectorySeparatorChar, '_'), $"{downloadInfo.ResolvedSha}.tar.gz"); #endif - if (File.Exists(cacheArchiveFile)) + if (File.Exists(cacheArchiveFile)) + { + try { - try - { - Trace.Info($"Found action archive '{cacheArchiveFile}' in cache directory '{actionArchiveCacheDir}'"); - File.Copy(cacheArchiveFile, archiveFile); - useActionArchiveCache = true; - executionContext.Debug($"Copied action archive '{cacheArchiveFile}' to '{archiveFile}'"); - } - catch (Exception ex) - { - Trace.Error($"Failed to copy action archive '{cacheArchiveFile}' to '{archiveFile}'. Error: {ex}"); - } + Trace.Info($"Found action archive '{cacheArchiveFile}' in cache directory '{actionArchiveCacheDir}'"); + File.Copy(cacheArchiveFile, archiveFile); + useActionArchiveCache = true; + executionContext.Debug($"Copied action archive '{cacheArchiveFile}' to '{archiveFile}'"); + } + catch (Exception ex) + { + Trace.Error($"Failed to copy action archive '{cacheArchiveFile}' to '{archiveFile}'. Error: {ex}"); } } - - executionContext.Global.JobTelemetry.Add(new JobTelemetry() - { - Type = JobTelemetryType.General, - Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}" - }); } + executionContext.Global.JobTelemetry.Add(new JobTelemetry() + { + Type = JobTelemetryType.General, + Message = $"Action archive cache usage: {downloadInfo.ResolvedNameWithOwner}@{downloadInfo.ResolvedSha} use cache {useActionArchiveCache} has cache {hasActionArchiveCache}" + }); + if (!useActionArchiveCache) { await DownloadRepositoryArchive(executionContext, link, downloadInfo.Authentication?.Token, archiveFile); @@ -879,16 +876,9 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont int exitCode = await processInvoker.ExecuteAsync(stagingDirectory, tar, $"-xzf \"{archiveFile}\"", null, executionContext.CancellationToken); if (exitCode != 0) { - if (executionContext.Global.Variables.GetBoolean("DistributedTask.DetailUntarFailure") == true) - { - var fileInfo = new FileInfo(archiveFile); - var sha256hash = await IOUtil.GetFileContentSha256HashAsync(archiveFile); - throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile} (SHA256 '{sha256hash}', size '{fileInfo.Length}' bytes, tar outputs '{string.Join(' ', tarOutputs)}'). Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}."); - } - else - { - throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}."); - } + var fileInfo = new FileInfo(archiveFile); + var sha256hash = await IOUtil.GetFileContentSha256HashAsync(archiveFile); + throw new InvalidActionArchiveException($"Can't use 'tar -xzf' extract archive file: {archiveFile} (SHA256 '{sha256hash}', size '{fileInfo.Length}' bytes, tar outputs '{string.Join(' ', tarOutputs)}'). Action being checked out: {downloadInfo.NameWithOwner}@{downloadInfo.Ref}. return code: {exitCode}."); } } #endif diff --git a/src/Runner.Worker/Handlers/ScriptHandler.cs b/src/Runner.Worker/Handlers/ScriptHandler.cs index 30114f27c33..e6fa90a0a11 100644 --- a/src/Runner.Worker/Handlers/ScriptHandler.cs +++ b/src/Runner.Worker/Handlers/ScriptHandler.cs @@ -83,40 +83,19 @@ protected override void PrintActionDetails(ActionRunStage stage) shellCommand = "pwsh"; if (validateShellOnHost) { - if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true) - { - shellCommandPath = WhichUtil.Which2(shellCommand, require: false, Trace, prependPath); - } - else - { - shellCommandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath); - } + shellCommandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath); if (string.IsNullOrEmpty(shellCommandPath)) { shellCommand = "powershell"; - Trace.Info($"Defaulting to {shellCommand}"); - if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true) - { - shellCommandPath = WhichUtil.Which2(shellCommand, require: true, Trace, prependPath); - } - else - { - shellCommandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath); - } + Trace.Info($"Defaulting to {shellCommand}"); + shellCommandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath); } } #else shellCommand = "sh"; if (validateShellOnHost) { - if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true) - { - shellCommandPath = WhichUtil.Which2("bash", false, Trace, prependPath) ?? WhichUtil.Which2("sh", true, Trace, prependPath); - } - else - { - shellCommandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath); - } + shellCommandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath); } #endif argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand); @@ -127,14 +106,7 @@ protected override void PrintActionDetails(ActionRunStage stage) shellCommand = parsed.shellCommand; if (validateShellOnHost) { - if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true) - { - shellCommandPath = WhichUtil.Which2(parsed.shellCommand, true, Trace, prependPath); - } - else - { - shellCommandPath = WhichUtil.Which(parsed.shellCommand, true, Trace, prependPath); - } + shellCommandPath = WhichUtil.Which(parsed.shellCommand, true, Trace, prependPath); } argFormat = $"{parsed.shellArgs}".TrimStart(); @@ -216,38 +188,17 @@ public async Task RunAsync(ActionRunStage stage) { #if OS_WINDOWS shellCommand = "pwsh"; - if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true) - { - commandPath = WhichUtil.Which2(shellCommand, require: false, Trace, prependPath); - } - else - { - commandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath); - } + commandPath = WhichUtil.Which(shellCommand, require: false, Trace, prependPath); if (string.IsNullOrEmpty(commandPath)) { shellCommand = "powershell"; Trace.Info($"Defaulting to {shellCommand}"); - if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true) - { - commandPath = WhichUtil.Which2(shellCommand, require: true, Trace, prependPath); - } - else - { - commandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath); - } + commandPath = WhichUtil.Which(shellCommand, require: true, Trace, prependPath); } ArgUtil.NotNullOrEmpty(commandPath, "Default Shell"); #else shellCommand = "sh"; - if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true) - { - commandPath = WhichUtil.Which2("bash", false, Trace, prependPath) ?? WhichUtil.Which2("sh", true, Trace, prependPath); - } - else - { - commandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath); - } + commandPath = WhichUtil.Which("bash", false, Trace, prependPath) ?? WhichUtil.Which("sh", true, Trace, prependPath); #endif argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat(shellCommand); } @@ -258,14 +209,7 @@ public async Task RunAsync(ActionRunStage stage) if (!IsActionStep && systemShells.Contains(shell)) { shellCommand = shell; - if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true) - { - commandPath = WhichUtil.Which2(shell, !isContainerStepHost, Trace, prependPath); - } - else - { - commandPath = WhichUtil.Which(shell, !isContainerStepHost, Trace, prependPath); - } + commandPath = WhichUtil.Which(shell, !isContainerStepHost, Trace, prependPath); if (shell == "bash") { argFormat = ScriptHandlerHelpers.GetScriptArgumentsFormat("sh"); @@ -280,14 +224,7 @@ public async Task RunAsync(ActionRunStage stage) var parsed = ScriptHandlerHelpers.ParseShellOptionString(shell); shellCommand = parsed.shellCommand; // For non-ContainerStepHost, the command must be located on the host by Which - if (ExecutionContext.Global.Variables.GetBoolean("DistributedTask.UseWhich2") == true) - { - commandPath = WhichUtil.Which2(parsed.shellCommand, !isContainerStepHost, Trace, prependPath); - } - else - { - commandPath = WhichUtil.Which(parsed.shellCommand, !isContainerStepHost, Trace, prependPath); - } + commandPath = WhichUtil.Which(parsed.shellCommand, !isContainerStepHost, Trace, prependPath); argFormat = $"{parsed.shellArgs}".TrimStart(); if (string.IsNullOrEmpty(argFormat)) { diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 7edff9ac363..ad265ecaf68 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -55,13 +55,6 @@ public async Task RunAsync(AgentJobRequestMessage message, Cancellat VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy); } - var jobServerQueueTelemetry = false; - if (message.Variables.TryGetValue("DistributedTask.EnableJobServerQueueTelemetry", out VariableValue enableJobServerQueueTelemetry) && - !string.IsNullOrEmpty(enableJobServerQueueTelemetry?.Value)) - { - jobServerQueueTelemetry = StringUtil.ConvertToBoolean(enableJobServerQueueTelemetry.Value); - } - ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); if (MessageUtil.IsRunServiceJob(message.MessageType)) { @@ -83,7 +76,7 @@ public async Task RunAsync(AgentJobRequestMessage message, Cancellat launchServer.InitializeLaunchClient(new Uri(launchReceiverEndpoint), accessToken); } _jobServerQueue = HostContext.GetService(); - _jobServerQueue.Start(message, resultsServiceOnly: true, enableTelemetry: jobServerQueueTelemetry); + _jobServerQueue.Start(message, resultsServiceOnly: true); } else { @@ -105,7 +98,7 @@ public async Task RunAsync(AgentJobRequestMessage message, Cancellat VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, delegatingHandlers); await jobServer.ConnectAsync(jobConnection); - _jobServerQueue.Start(message, enableTelemetry: jobServerQueueTelemetry); + _jobServerQueue.Start(message); server = jobServer; } diff --git a/src/Test/L0/Util/WhichUtilL0.cs b/src/Test/L0/Util/WhichUtilL0.cs index 90d32c466a6..9a6443d1fff 100644 --- a/src/Test/L0/Util/WhichUtilL0.cs +++ b/src/Test/L0/Util/WhichUtilL0.cs @@ -212,210 +212,5 @@ public void WhichThrowsWhenSymlinkBroken() File.Delete(brokenSymlink); Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue); } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Common")] - public void UseWhich2FindGit() - { - using (TestHostContext hc = new(this)) - { - //Arrange - Tracing trace = hc.GetTrace(); - - // Act. - string gitPath = WhichUtil.Which2("git", trace: trace); - - trace.Info($"Which(\"git\") returns: {gitPath ?? string.Empty}"); - - // Assert. - Assert.True(!string.IsNullOrEmpty(gitPath) && File.Exists(gitPath), $"Unable to find Git through: {nameof(WhichUtil.Which)}"); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Common")] - public void Which2ReturnsNullWhenNotFound() - { - using (TestHostContext hc = new(this)) - { - //Arrange - Tracing trace = hc.GetTrace(); - - // Act. - string nosuch = WhichUtil.Which2("no-such-file-cf7e351f", trace: trace); - - trace.Info($"result: {nosuch ?? string.Empty}"); - - // Assert. - Assert.True(string.IsNullOrEmpty(nosuch), "Path should not be resolved"); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Common")] - public void Which2ThrowsWhenRequireAndNotFound() - { - using (TestHostContext hc = new(this)) - { - //Arrange - Tracing trace = hc.GetTrace(); - - // Act. - try - { - WhichUtil.Which2("no-such-file-cf7e351f", require: true, trace: trace); - throw new Exception("which should have thrown"); - } - catch (FileNotFoundException ex) - { - Assert.Equal("no-such-file-cf7e351f", ex.FileName); - } - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Common")] - public void Which2HandleFullyQualifiedPath() - { - using (TestHostContext hc = new(this)) - { - //Arrange - Tracing trace = hc.GetTrace(); - - // Act. - var gitPath = WhichUtil.Which2("git", require: true, trace: trace); - var gitPath2 = WhichUtil.Which2(gitPath, require: true, trace: trace); - - // Assert. - Assert.Equal(gitPath, gitPath2); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Common")] - public void Which2HandlesSymlinkToTargetFullPath() - { - // Arrange - using TestHostContext hc = new TestHostContext(this); - Tracing trace = hc.GetTrace(); - string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable); -#if OS_WINDOWS - string newValue = oldValue + @$";{Path.GetTempPath()}"; - string symlinkName = $"symlink-{Guid.NewGuid()}"; - string symlink = Path.GetTempPath() + $"{symlinkName}.exe"; - string target = Path.GetTempPath() + $"target-{Guid.NewGuid()}.exe"; -#else - string newValue = oldValue + @$":{Path.GetTempPath()}"; - string symlinkName = $"symlink-{Guid.NewGuid()}"; - string symlink = Path.GetTempPath() + $"{symlinkName}"; - string target = Path.GetTempPath() + $"target-{Guid.NewGuid()}"; -#endif - - Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue); - - - using (File.Create(target)) - { - File.CreateSymbolicLink(symlink, target); - - // Act. - var result = WhichUtil.Which2(symlinkName, require: true, trace: trace); - - // Assert - Assert.True(!string.IsNullOrEmpty(result) && File.Exists(result), $"Unable to find symlink through: {nameof(WhichUtil.Which)}"); - - } - - - // Cleanup - File.Delete(symlink); - File.Delete(target); - Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue); - - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Common")] - public void Which2HandlesSymlinkToTargetRelativePath() - { - // Arrange - using TestHostContext hc = new TestHostContext(this); - Tracing trace = hc.GetTrace(); - string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable); -#if OS_WINDOWS - string newValue = oldValue + @$";{Path.GetTempPath()}"; - string symlinkName = $"symlink-{Guid.NewGuid()}"; - string symlink = Path.GetTempPath() + $"{symlinkName}.exe"; - string targetName = $"target-{Guid.NewGuid()}.exe"; - string target = Path.GetTempPath() + targetName; -#else - string newValue = oldValue + @$":{Path.GetTempPath()}"; - string symlinkName = $"symlink-{Guid.NewGuid()}"; - string symlink = Path.GetTempPath() + $"{symlinkName}"; - string targetName = $"target-{Guid.NewGuid()}"; - string target = Path.GetTempPath() + targetName; -#endif - Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue); - - - using (File.Create(target)) - { - File.CreateSymbolicLink(symlink, targetName); - - // Act. - var result = WhichUtil.Which2(symlinkName, require: true, trace: trace); - - // Assert - Assert.True(!string.IsNullOrEmpty(result) && File.Exists(result), $"Unable to find {symlinkName} through: {nameof(WhichUtil.Which)}"); - } - - // Cleanup - File.Delete(symlink); - File.Delete(target); - Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue); - - } - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Common")] - public void Which2ThrowsWhenSymlinkBroken() - { - // Arrange - using TestHostContext hc = new TestHostContext(this); - Tracing trace = hc.GetTrace(); - string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable); - -#if OS_WINDOWS - string newValue = oldValue + @$";{Path.GetTempPath()}"; - string brokenSymlinkName = $"broken-symlink-{Guid.NewGuid()}"; - string brokenSymlink = Path.GetTempPath() + $"{brokenSymlinkName}.exe"; -#else - string newValue = oldValue + @$":{Path.GetTempPath()}"; - string brokenSymlinkName = $"broken-symlink-{Guid.NewGuid()}"; - string brokenSymlink = Path.GetTempPath() + $"{brokenSymlinkName}"; -#endif - - - string target = "no-such-file-cf7e351f"; - Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue); - - File.CreateSymbolicLink(brokenSymlink, target); - - // Act. - var exception = Assert.Throws(() => WhichUtil.Which2(brokenSymlinkName, require: true, trace: trace)); - - // Assert - Assert.Equal(brokenSymlinkName, exception.FileName); - - // Cleanup - File.Delete(brokenSymlink); - Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue); - } } } diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index c487ea55ef1..d4430d91128 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -382,8 +382,6 @@ public async void PrepareActions_DownloadActionFromGraph_UseCache() } }; - _ec.Object.Global.Variables.Set("DistributedTask.UseActionArchiveCache", bool.TrueString); - //Act await _actionManager.PrepareActionsAsync(_ec.Object, actions); From 1b61d78c07af22165b8f563cf52181e8399668b4 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Wed, 17 Apr 2024 09:55:03 -0400 Subject: [PATCH 18/68] Relax the condition to stop uploading to Results (#3230) --- src/Runner.Common/JobServerQueue.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 9c27bf51d0e..e489ff2029f 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -74,6 +74,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue private readonly List _jobTelemetries = new(); private bool _queueInProcess = false; private bool _resultsServiceOnly = false; + private int _resultsServiceExceptionsCount = 0; private Stopwatch _resultsUploadTimer = new(); private Stopwatch _actionsUploadTimer = new(); @@ -579,9 +580,9 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) Trace.Info("Catch exception during file upload to results, keep going since the process is best effort."); Trace.Error(ex); errorCount++; - + _resultsServiceExceptionsCount++; // If we hit any exceptions uploading to Results, let's skip any additional uploads to Results unless Results is serving logs - if (!_resultsServiceOnly) + if (!_resultsServiceOnly && _resultsServiceExceptionsCount > 3) { _resultsClientInitiated = false; SendResultsTelemetry(ex); From 5107c5efb2b122903d95f59d492dfc9438419230 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Fri, 19 Apr 2024 15:31:44 -0400 Subject: [PATCH 19/68] Cleanup enabled feature flags. (#3248) --- src/Runner.Listener/JobDispatcher.cs | 13 ++++--------- src/Runner.Worker/ActionManager.cs | 11 ----------- src/Test/L0/Worker/ActionManagerL0.cs | 4 ---- 3 files changed, 4 insertions(+), 24 deletions(-) diff --git a/src/Runner.Listener/JobDispatcher.cs b/src/Runner.Listener/JobDispatcher.cs index ef664936ea8..ede26db1810 100644 --- a/src/Runner.Listener/JobDispatcher.cs +++ b/src/Runner.Listener/JobDispatcher.cs @@ -1155,18 +1155,13 @@ private async Task LogWorkerProcessUnhandledException(IJobServer jobServer, Pipe TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job"); ArgUtil.NotNull(jobRecord, nameof(jobRecord)); - jobRecord.ErrorCount++; jobRecord.Issues.Add(issue); - if (message.Variables.TryGetValue("DistributedTask.MarkJobAsFailedOnWorkerCrash", out var markJobAsFailedOnWorkerCrash) && - StringUtil.ConvertToBoolean(markJobAsFailedOnWorkerCrash?.Value)) - { - Trace.Info("Mark the job as failed since the worker crashed"); - jobRecord.Result = TaskResult.Failed; - // mark the job as completed so service will pickup the result - jobRecord.State = TimelineRecordState.Completed; - } + Trace.Info("Mark the job as failed since the worker crashed"); + jobRecord.Result = TaskResult.Failed; + // mark the job as completed so service will pickup the result + jobRecord.State = TimelineRecordState.Completed; await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None); } diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 227e7f4a5e6..0ec72b0585f 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -483,10 +483,6 @@ public Definition LoadAction(IExecutionContext executionContext, Pipelines.Actio { // Load stored Ids for later load actions compositeAction.Steps[i].Id = _cachedEmbeddedStepIds[action.Id][i]; - if (string.IsNullOrEmpty(executionContext.Global.Variables.Get("DistributedTask.EnableCompositeActions")) && compositeAction.Steps[i].Reference.Type != Pipelines.ActionSourceType.Script) - { - throw new Exception("`uses:` keyword is not currently supported."); - } } } else @@ -1022,13 +1018,6 @@ private ActionSetupInfo PrepareRepositoryActionAsync(IExecutionContext execution } } - foreach (var step in compositeAction.Steps) - { - if (string.IsNullOrEmpty(executionContext.Global.Variables.Get("DistributedTask.EnableCompositeActions")) && step.Reference.Type != Pipelines.ActionSourceType.Script) - { - throw new Exception("`uses:` keyword is not currently supported."); - } - } return setupInfo; } else diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index d4430d91128..1176966149f 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -2373,10 +2373,6 @@ private void Setup([CallerMemberName] string name = "", bool enableComposite = t _ec.Setup(x => x.CancellationToken).Returns(_ecTokenSource.Token); _ec.Setup(x => x.Root).Returns(new GitHub.Runner.Worker.ExecutionContext()); var variables = new Dictionary(); - if (enableComposite) - { - variables["DistributedTask.EnableCompositeActions"] = "true"; - } _ec.Object.Global.Variables = new Variables(_hc, variables); _ec.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData()); _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); From dd9fcfc5b267a62720db734e660514071184eb5e Mon Sep 17 00:00:00 2001 From: eric sciple Date: Sat, 20 Apr 2024 11:37:25 -0500 Subject: [PATCH 20/68] Replace invalid file name chars in diag log name (#3249) --- src/Runner.Sdk/Util/IOUtil.cs | 28 +++++++++++++++++++++++ src/Runner.Worker/DiagnosticLogManager.cs | 4 ++-- src/Test/L0/Util/IOUtilL0.cs | 27 ++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/Runner.Sdk/Util/IOUtil.cs b/src/Runner.Sdk/Util/IOUtil.cs index da4a8a09bcf..e0b5b3394f7 100644 --- a/src/Runner.Sdk/Util/IOUtil.cs +++ b/src/Runner.Sdk/Util/IOUtil.cs @@ -459,6 +459,34 @@ public static void CreateEmptyFile(string path) File.WriteAllText(path, null); } + /// + /// Replaces invalid file name characters with '_' + /// + public static string ReplaceInvalidFileNameChars(string fileName) + { + var result = new StringBuilder(); + var invalidChars = Path.GetInvalidFileNameChars(); + + var current = 0; // Current index + while (current < fileName?.Length) + { + var next = fileName.IndexOfAny(invalidChars, current); + if (next >= 0) + { + result.Append(fileName.Substring(current, next - current)); + result.Append('_'); + current = next + 1; + } + else + { + result.Append(fileName.Substring(current)); + break; + } + } + + return result.ToString(); + } + /// /// Recursively enumerates a directory without following directory reparse points. /// diff --git a/src/Runner.Worker/DiagnosticLogManager.cs b/src/Runner.Worker/DiagnosticLogManager.cs index 261689b5f3f..afc811b15e4 100644 --- a/src/Runner.Worker/DiagnosticLogManager.cs +++ b/src/Runner.Worker/DiagnosticLogManager.cs @@ -91,13 +91,13 @@ public void UploadDiagnosticLogs(IExecutionContext executionContext, string phaseName = executionContext.Global.Variables.System_PhaseDisplayName ?? "UnknownPhaseName"; // zip the files - string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip"; + string diagnosticsZipFileName = $"{buildName}-{IOUtil.ReplaceInvalidFileNameChars(phaseName)}.zip"; string diagnosticsZipFilePath = Path.Combine(supportRootFolder, diagnosticsZipFileName); ZipFile.CreateFromDirectory(supportFilesFolder, diagnosticsZipFilePath); // upload the json metadata file executionContext.Debug("Uploading diagnostic metadata file."); - string metadataFileName = $"diagnostics-{buildName}-{phaseName}.json"; + string metadataFileName = $"diagnostics-{buildName}-{IOUtil.ReplaceInvalidFileNameChars(phaseName)}.json"; string metadataFilePath = Path.Combine(supportFilesFolder, metadataFileName); string phaseResult = GetTaskResultAsString(executionContext.Result); diff --git a/src/Test/L0/Util/IOUtilL0.cs b/src/Test/L0/Util/IOUtilL0.cs index 08d3e9773d1..e9478dff210 100644 --- a/src/Test/L0/Util/IOUtilL0.cs +++ b/src/Test/L0/Util/IOUtilL0.cs @@ -960,6 +960,33 @@ public void LoadObject_ThrowsOnRequiredLoadObject() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void ReplaceInvalidFileNameChars() + { + Assert.Equal(string.Empty, IOUtil.ReplaceInvalidFileNameChars(null)); + Assert.Equal(string.Empty, IOUtil.ReplaceInvalidFileNameChars(string.Empty)); + Assert.Equal("hello.txt", IOUtil.ReplaceInvalidFileNameChars("hello.txt")); +#if OS_WINDOWS + // Refer https://github.com/dotnet/runtime/blob/ce84f1d8a3f12711bad678a33efbc37b461f684f/src/libraries/System.Private.CoreLib/src/System/IO/Path.Windows.cs#L15 + Assert.Equal( + "1_ 2_ 3_ 4_ 5_ 6_ 7_ 8_ 9_ 10_ 11_ 12_ 13_ 14_ 15_ 16_ 17_ 18_ 19_ 20_ 21_ 22_ 23_ 24_ 25_ 26_ 27_ 28_ 29_ 30_ 31_ 32_ 33_ 34_ 35_ 36_ 37_ 38_ 39_ 40_ 41_", + IOUtil.ReplaceInvalidFileNameChars($"1\" 2< 3> 4| 5\0 6{(char)1} 7{(char)2} 8{(char)3} 9{(char)4} 10{(char)5} 11{(char)6} 12{(char)7} 13{(char)8} 14{(char)9} 15{(char)10} 16{(char)11} 17{(char)12} 18{(char)13} 19{(char)14} 20{(char)15} 21{(char)16} 22{(char)17} 23{(char)18} 24{(char)19} 25{(char)20} 26{(char)21} 27{(char)22} 28{(char)23} 29{(char)24} 30{(char)25} 31{(char)26} 32{(char)27} 33{(char)28} 34{(char)29} 35{(char)30} 36{(char)31} 37: 38* 39? 40\\ 41/")); +#else + // Refer https://github.com/dotnet/runtime/blob/ce84f1d8a3f12711bad678a33efbc37b461f684f/src/libraries/System.Private.CoreLib/src/System/IO/Path.Unix.cs#L12 + Assert.Equal("1_ 2_", IOUtil.ReplaceInvalidFileNameChars("1\0 2/")); +#endif + Assert.Equal("_leading", IOUtil.ReplaceInvalidFileNameChars("/leading")); + Assert.Equal("__consecutive leading", IOUtil.ReplaceInvalidFileNameChars("//consecutive leading")); + Assert.Equal("trailing_", IOUtil.ReplaceInvalidFileNameChars("trailing/")); + Assert.Equal("consecutive trailing__", IOUtil.ReplaceInvalidFileNameChars("consecutive trailing//")); + Assert.Equal("middle_middle", IOUtil.ReplaceInvalidFileNameChars("middle/middle")); + Assert.Equal("consecutive middle__consecutive middle", IOUtil.ReplaceInvalidFileNameChars("consecutive middle//consecutive middle")); + Assert.Equal("_leading_middle_trailing_", IOUtil.ReplaceInvalidFileNameChars("/leading/middle/trailing/")); + Assert.Equal("__consecutive leading__consecutive middle__consecutive trailing__", IOUtil.ReplaceInvalidFileNameChars("//consecutive leading//consecutive middle//consecutive trailing//")); + } + private static async Task CreateDirectoryReparsePoint(IHostContext context, string link, string target) { #if OS_WINDOWS From 04b07b6675c56da40e532ab73eeb4287bf223e34 Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Tue, 23 Apr 2024 16:46:17 +0100 Subject: [PATCH 21/68] Prepare v2.316.0 release (#3252) --- releaseNote.md | 27 +++++++++++---------------- src/runnerversion | 2 +- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index cf0c0688ca5..7bbd49af9dc 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,23 +1,18 @@ ## What's Changed -* fix summaries for actions results by @SrRyan in https://github.com/actions/runner/pull/3174 -* Bump runner version to match the latest patch release by @TingluoHuang in https://github.com/actions/runner/pull/3175 -* don't crash listener on getting job exceptions for run-service by @yaananth in https://github.com/actions/runner/pull/3177 -* Remove -f flag in wait when manually trap signal by @nikola-jokic in https://github.com/actions/runner/pull/3182 -* consume new pipelines service url in handlers by @patrickcarnahan in https://github.com/actions/runner/pull/3185 -* Add ability to enforce actions to run on node20 by @takost in https://github.com/actions/runner/pull/3192 -* Bump hook version to 0.6.0 by @nikola-jokic in https://github.com/actions/runner/pull/3203 -* Update dotnet sdk to latest version @6.0.420 by @github-actions in https://github.com/actions/runner/pull/3211 -* Bump docker version and docker buildx version by @nikola-jokic in https://github.com/actions/runner/pull/3208 -* Handle new non-retryable exception type by @thyeggman in https://github.com/actions/runner/pull/3191 -* Always Delete Actions Service Session by @luketomlinson in https://github.com/actions/runner/pull/3214 +* Load '_runnerSettings' in the early point of JobRunner.cs by @TingluoHuang in https://github.com/actions/runner/pull/3218 +* Add new SessionConflict return code by @eeSquared in https://github.com/actions/runner/pull/3215 +* backoff if we retried polling for more than 50 times in less than 30minutes by @aiqiaoy in https://github.com/actions/runner/pull/3232 +* Update dotnet sdk to latest version @6.0.421 by @github-actions in https://github.com/actions/runner/pull/3244 +* Cleanup enabled feature flags. by @TingluoHuang in https://github.com/actions/runner/pull/3246 +* Relax the condition to stop uploading to Results by @yacaovsnc in https://github.com/actions/runner/pull/3230 +* Cleanup enabled feature flags. by @TingluoHuang in https://github.com/actions/runner/pull/3248 +* Replace invalid file name chars in diag log name by @ericsciple in https://github.com/actions/runner/pull/3249 ## New Contributors -* @SrRyan made their first contribution in https://github.com/actions/runner/pull/3174 -* @patrickcarnahan made their first contribution in https://github.com/actions/runner/pull/3185 +* @eeSquared made their first contribution in https://github.com/actions/runner/pull/3215 +* @aiqiaoy made their first contribution in https://github.com/actions/runner/pull/3232 -**Full Changelog**: https://github.com/actions/runner/compare/v2.314.1...v2.315.0 - -**Full Changelog**: https://github.com/actions/runner/compare/v2.313.0...v2.314.0 +**Full Changelog**: https://github.com/actions/runner/compare/v2.315.0...v2.316.0 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. diff --git a/src/runnerversion b/src/runnerversion index bb8021df34e..ce7c32e00ce 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.315.0 +2.316.0 From 18803bdff6795829d3bc63d0846691e9ac0a4e21 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Thu, 2 May 2024 09:44:57 -0500 Subject: [PATCH 22/68] Preserve dates when deserializing job message from Run Service (#3269) * Preserve dates when deserializing job message from Run Service * Preserve dates when deserializing job message from "Actions Run Service" --- src/Runner.Common/ActionsRunServer.cs | 6 +- .../WebApi/ActionsRunServerHttpClient.cs | 95 +++++++++++++++++++ .../DTWebApi/WebApi/TaskAgentHttpClient.cs | 18 ---- src/Sdk/RSWebApi/RunServiceHttpClient.cs | 16 ++++ 4 files changed, 114 insertions(+), 21 deletions(-) create mode 100644 src/Sdk/DTWebApi/WebApi/ActionsRunServerHttpClient.cs diff --git a/src/Runner.Common/ActionsRunServer.cs b/src/Runner.Common/ActionsRunServer.cs index 704a690e3ca..3ded580071d 100644 --- a/src/Runner.Common/ActionsRunServer.cs +++ b/src/Runner.Common/ActionsRunServer.cs @@ -20,12 +20,12 @@ public sealed class ActionsRunServer : RunnerService, IActionsRunServer { private bool _hasConnection; private VssConnection _connection; - private TaskAgentHttpClient _taskAgentClient; + private ActionsRunServerHttpClient _actionsRunServerClient; public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials) { _connection = await EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100)); - _taskAgentClient = _connection.GetClient(); + _actionsRunServerClient = _connection.GetClient(); _hasConnection = true; } @@ -42,7 +42,7 @@ public Task GetJobMessageAsync(string id, CancellationTo CheckConnection(); var jobMessage = RetryRequest(async () => { - return await _taskAgentClient.GetJobMessageAsync(id, cancellationToken); + return await _actionsRunServerClient.GetJobMessageAsync(id, cancellationToken); }, cancellationToken); return jobMessage; diff --git a/src/Sdk/DTWebApi/WebApi/ActionsRunServerHttpClient.cs b/src/Sdk/DTWebApi/WebApi/ActionsRunServerHttpClient.cs new file mode 100644 index 00000000000..a72e8a28bbf --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/ActionsRunServerHttpClient.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Services.Common; +using GitHub.Services.Common.Diagnostics; +using GitHub.Services.WebApi; +using Newtonsoft.Json; + +namespace GitHub.DistributedTask.WebApi +{ + [ResourceArea(TaskResourceIds.AreaId)] + public class ActionsRunServerHttpClient : TaskAgentHttpClient + { + private static readonly JsonSerializerSettings s_serializerSettings; + + static ActionsRunServerHttpClient() + { + s_serializerSettings = new VssJsonMediaTypeFormatter().SerializerSettings; + s_serializerSettings.DateParseHandling = DateParseHandling.None; + s_serializerSettings.FloatParseHandling = FloatParseHandling.Double; + } + + public ActionsRunServerHttpClient( + Uri baseUrl, + VssCredentials credentials) + : base(baseUrl, credentials) + { + } + + public ActionsRunServerHttpClient( + Uri baseUrl, + VssCredentials credentials, + VssHttpRequestSettings settings) + : base(baseUrl, credentials, settings) + { + } + + public ActionsRunServerHttpClient( + Uri baseUrl, + VssCredentials credentials, + params DelegatingHandler[] handlers) + : base(baseUrl, credentials, handlers) + { + } + + public ActionsRunServerHttpClient( + Uri baseUrl, + VssCredentials credentials, + VssHttpRequestSettings settings, + params DelegatingHandler[] handlers) + : base(baseUrl, credentials, settings, handlers) + { + } + + public ActionsRunServerHttpClient( + Uri baseUrl, + HttpMessageHandler pipeline, + Boolean disposeHandler) + : base(baseUrl, pipeline, disposeHandler) + { + } + + public Task GetJobMessageAsync( + string messageId, + object userState = null, + CancellationToken cancellationToken = default) + { + HttpMethod httpMethod = new HttpMethod("GET"); + Guid locationId = new Guid("25adab70-1379-4186-be8e-b643061ebe3a"); + object routeValues = new { messageId = messageId }; + + return SendAsync( + httpMethod, + locationId, + routeValues: routeValues, + version: new ApiResourceVersion(6.0, 1), + userState: userState, + cancellationToken: cancellationToken); + } + + protected override async Task ReadJsonContentAsync(HttpResponseMessage response, CancellationToken cancellationToken = default(CancellationToken)) + { + var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return JsonConvert.DeserializeObject(json, s_serializerSettings); + } + } +} diff --git a/src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs b/src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs index 4b08ebaa83a..c97fea0a4d1 100644 --- a/src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs +++ b/src/Sdk/DTWebApi/WebApi/TaskAgentHttpClient.cs @@ -141,24 +141,6 @@ public Task ReplaceAgentAsync( return ReplaceAgentAsync(poolId, agent.Id, agent, userState, cancellationToken); } - public Task GetJobMessageAsync( - string messageId, - object userState = null, - CancellationToken cancellationToken = default) - { - HttpMethod httpMethod = new HttpMethod("GET"); - Guid locationId = new Guid("25adab70-1379-4186-be8e-b643061ebe3a"); - object routeValues = new { messageId = messageId }; - - return SendAsync( - httpMethod, - locationId, - routeValues: routeValues, - version: new ApiResourceVersion(6.0, 1), - userState: userState, - cancellationToken: cancellationToken); - } - protected Task SendAsync( HttpMethod method, Guid locationId, diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index 4d2b74f8c4c..ba176ccf6b6 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -9,6 +9,7 @@ using GitHub.Services.Common; using GitHub.Services.OAuth; using GitHub.Services.WebApi; +using Newtonsoft.Json; using Sdk.RSWebApi.Contracts; using Sdk.WebApi.WebApi; @@ -16,6 +17,15 @@ namespace GitHub.Actions.RunService.WebApi { public class RunServiceHttpClient : RawHttpClientBase { + private static readonly JsonSerializerSettings s_serializerSettings; + + static RunServiceHttpClient() + { + s_serializerSettings = new VssJsonMediaTypeFormatter().SerializerSettings; + s_serializerSettings.DateParseHandling = DateParseHandling.None; + s_serializerSettings.FloatParseHandling = FloatParseHandling.Double; + } + public RunServiceHttpClient( Uri baseUrl, VssOAuthCredential credentials) @@ -174,5 +184,11 @@ public async Task RenewJobAsync( throw new Exception($"Failed to renew job: {result.Error}"); } } + + protected override async Task ReadJsonContentAsync(HttpResponseMessage response, CancellationToken cancellationToken = default(CancellationToken)) + { + var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return JsonConvert.DeserializeObject(json, s_serializerSettings); + } } } From f2c05de91c8a6448e86772d3c818c0d0e5e32474 Mon Sep 17 00:00:00 2001 From: Luke Tomlinson Date: Thu, 2 May 2024 13:44:48 -0400 Subject: [PATCH 23/68] Prep 2.316.1 Release (#3272) --- releaseNote.md | 33 ++++++++++++++------------------- src/runnerversion | 2 +- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 7bbd49af9dc..067b48bd66c 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,28 +1,20 @@ ## What's Changed -* Load '_runnerSettings' in the early point of JobRunner.cs by @TingluoHuang in https://github.com/actions/runner/pull/3218 -* Add new SessionConflict return code by @eeSquared in https://github.com/actions/runner/pull/3215 -* backoff if we retried polling for more than 50 times in less than 30minutes by @aiqiaoy in https://github.com/actions/runner/pull/3232 -* Update dotnet sdk to latest version @6.0.421 by @github-actions in https://github.com/actions/runner/pull/3244 -* Cleanup enabled feature flags. by @TingluoHuang in https://github.com/actions/runner/pull/3246 -* Relax the condition to stop uploading to Results by @yacaovsnc in https://github.com/actions/runner/pull/3230 -* Cleanup enabled feature flags. by @TingluoHuang in https://github.com/actions/runner/pull/3248 -* Replace invalid file name chars in diag log name by @ericsciple in https://github.com/actions/runner/pull/3249 -## New Contributors -* @eeSquared made their first contribution in https://github.com/actions/runner/pull/3215 -* @aiqiaoy made their first contribution in https://github.com/actions/runner/pull/3232 +- Preserve dates when deserializing job message from Run Service by @ericsciple in https://github.com/actions/runner/pull/3269 -**Full Changelog**: https://github.com/actions/runner/compare/v2.315.0...v2.316.0 +**Full Changelog**: https://github.com/actions/runner/compare/v2.316.0...v2.316.1 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. See https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/adding-self-hosted-runners_ ## Windows x64 + We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. The following snipped needs to be run on `powershell`: -``` powershell + +```powershell # Create a folder under the drive root mkdir \actions-runner ; cd \actions-runner # Download the latest runner package @@ -33,12 +25,14 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ; ``` ## [Pre-release] Windows arm64 + **Warning:** Windows arm64 runners are currently in preview status and use [unofficial versions of nodejs](https://unofficial-builds.nodejs.org/). They are not intended for production workflows. We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. The following snipped needs to be run on `powershell`: -``` powershell + +```powershell # Create a folder under the drive root mkdir \actions-runner ; cd \actions-runner # Download the latest runner package @@ -50,7 +44,7 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ; ## OSX x64 -``` bash +```bash # Create a folder mkdir actions-runner && cd actions-runner # Download the latest runner package @@ -61,7 +55,7 @@ tar xzf ./actions-runner-osx-x64-.tar.gz ## OSX arm64 (Apple silicon) -``` bash +```bash # Create a folder mkdir actions-runner && cd actions-runner # Download the latest runner package @@ -72,7 +66,7 @@ tar xzf ./actions-runner-osx-arm64-.tar.gz ## Linux x64 -``` bash +```bash # Create a folder mkdir actions-runner && cd actions-runner # Download the latest runner package @@ -83,7 +77,7 @@ tar xzf ./actions-runner-linux-x64-.tar.gz ## Linux arm64 -``` bash +```bash # Create a folder mkdir actions-runner && cd actions-runner # Download the latest runner package @@ -94,7 +88,7 @@ tar xzf ./actions-runner-linux-arm64-.tar.gz ## Linux arm -``` bash +```bash # Create a folder mkdir actions-runner && cd actions-runner # Download the latest runner package @@ -104,6 +98,7 @@ tar xzf ./actions-runner-linux-arm-.tar.gz ``` ## Using your self hosted runner + For additional details about configuring, running, or shutting down the runner please check out our [product docs.](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/adding-self-hosted-runners) ## SHA-256 Checksums diff --git a/src/runnerversion b/src/runnerversion index ce7c32e00ce..ad3450ba108 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.316.0 +2.316.1 From 54052b94fb525a8bf30cc672c1ab103ac65297db Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Fri, 10 May 2024 11:02:29 -0400 Subject: [PATCH 24/68] Also do not give up when uploading steps metadata (#3280) --- src/Runner.Common/JobServerQueue.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index e489ff2029f..dd060734b79 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -709,7 +709,9 @@ private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) { Trace.Info("Catch exception during update steps, skip update Results."); Trace.Error(e); - if (!_resultsServiceOnly) + _resultsServiceExceptionsCount++; + // If we hit any exceptions uploading to Results, let's skip any additional uploads to Results unless Results is serving logs + if (!_resultsServiceOnly && _resultsServiceExceptionsCount > 3) { _resultsClientInitiated = false; SendResultsTelemetry(e); From c67e7f28130fd44cda176858b12229651ff19473 Mon Sep 17 00:00:00 2001 From: Patrick Ellis <319655+pje@users.noreply.github.com> Date: Mon, 13 May 2024 17:40:23 -0400 Subject: [PATCH 25/68] Delete all the contentHash files (#3285) Nothing uses them anymore after #3074. --- src/Misc/contentHash/dotnetRuntime/linux-arm | 1 - src/Misc/contentHash/dotnetRuntime/linux-arm64 | 1 - src/Misc/contentHash/dotnetRuntime/linux-x64 | 1 - src/Misc/contentHash/dotnetRuntime/osx-arm64 | 1 - src/Misc/contentHash/dotnetRuntime/osx-x64 | 1 - src/Misc/contentHash/dotnetRuntime/win-arm64 | 1 - src/Misc/contentHash/dotnetRuntime/win-x64 | 1 - src/Misc/contentHash/externals/linux-arm | 1 - src/Misc/contentHash/externals/linux-arm64 | 1 - src/Misc/contentHash/externals/linux-x64 | 1 - src/Misc/contentHash/externals/osx-arm64 | 1 - src/Misc/contentHash/externals/osx-x64 | 1 - src/Misc/contentHash/externals/win-arm64 | 1 - src/Misc/contentHash/externals/win-x64 | 1 - 14 files changed, 14 deletions(-) delete mode 100644 src/Misc/contentHash/dotnetRuntime/linux-arm delete mode 100644 src/Misc/contentHash/dotnetRuntime/linux-arm64 delete mode 100644 src/Misc/contentHash/dotnetRuntime/linux-x64 delete mode 100644 src/Misc/contentHash/dotnetRuntime/osx-arm64 delete mode 100644 src/Misc/contentHash/dotnetRuntime/osx-x64 delete mode 100644 src/Misc/contentHash/dotnetRuntime/win-arm64 delete mode 100644 src/Misc/contentHash/dotnetRuntime/win-x64 delete mode 100644 src/Misc/contentHash/externals/linux-arm delete mode 100644 src/Misc/contentHash/externals/linux-arm64 delete mode 100644 src/Misc/contentHash/externals/linux-x64 delete mode 100644 src/Misc/contentHash/externals/osx-arm64 delete mode 100644 src/Misc/contentHash/externals/osx-x64 delete mode 100644 src/Misc/contentHash/externals/win-arm64 delete mode 100644 src/Misc/contentHash/externals/win-x64 diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm b/src/Misc/contentHash/dotnetRuntime/linux-arm deleted file mode 100644 index 9f55d62ef2a..00000000000 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm +++ /dev/null @@ -1 +0,0 @@ -54d95a44d118dba852395991224a6b9c1abe916858c87138656f80c619e85331 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm64 b/src/Misc/contentHash/dotnetRuntime/linux-arm64 deleted file mode 100644 index c03c98ade6c..00000000000 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm64 +++ /dev/null @@ -1 +0,0 @@ -68015af17f06a824fa478e62ae7393766ce627fd5599ab916432a14656a19a52 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/linux-x64 b/src/Misc/contentHash/dotnetRuntime/linux-x64 deleted file mode 100644 index 95a7155f74d..00000000000 --- a/src/Misc/contentHash/dotnetRuntime/linux-x64 +++ /dev/null @@ -1 +0,0 @@ -a2628119ca419cb54e279103ffae7986cdbd0814d57c73ff0dc74c38be08b9ae \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/osx-arm64 b/src/Misc/contentHash/dotnetRuntime/osx-arm64 deleted file mode 100644 index d99ff5942f0..00000000000 --- a/src/Misc/contentHash/dotnetRuntime/osx-arm64 +++ /dev/null @@ -1 +0,0 @@ -de71ca09ead807e1a2ce9df0a5b23eb7690cb71fff51169a77e4c3992be53dda \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/osx-x64 b/src/Misc/contentHash/dotnetRuntime/osx-x64 deleted file mode 100644 index 085b329b2a0..00000000000 --- a/src/Misc/contentHash/dotnetRuntime/osx-x64 +++ /dev/null @@ -1 +0,0 @@ -d009e05e6b26d614d65be736a15d1bd151932121c16a9ff1b986deadecc982b9 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/win-arm64 b/src/Misc/contentHash/dotnetRuntime/win-arm64 deleted file mode 100644 index 5c84f556e8d..00000000000 --- a/src/Misc/contentHash/dotnetRuntime/win-arm64 +++ /dev/null @@ -1 +0,0 @@ -f730db39c2305800b4653795360ba9c10c68f384a46b85d808f1f9f0ed3c42e4 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/win-x64 b/src/Misc/contentHash/dotnetRuntime/win-x64 deleted file mode 100644 index 6be8253b146..00000000000 --- a/src/Misc/contentHash/dotnetRuntime/win-x64 +++ /dev/null @@ -1 +0,0 @@ -a35b5722375490e9473cdcccb5e18b41eba3dbf4344fe31abc9821e21f18ea5a \ No newline at end of file diff --git a/src/Misc/contentHash/externals/linux-arm b/src/Misc/contentHash/externals/linux-arm deleted file mode 100644 index 62be8089e13..00000000000 --- a/src/Misc/contentHash/externals/linux-arm +++ /dev/null @@ -1 +0,0 @@ -4bf3e1af0d482af1b2eaf9f08250248a8c1aea8ec20a3c5be116d58cdd930009 \ No newline at end of file diff --git a/src/Misc/contentHash/externals/linux-arm64 b/src/Misc/contentHash/externals/linux-arm64 deleted file mode 100644 index bde540d4f54..00000000000 --- a/src/Misc/contentHash/externals/linux-arm64 +++ /dev/null @@ -1 +0,0 @@ -ec1719a8cb4d8687328aa64f4aa7c4e3498a715d8939117874782e3e6e63a14b \ No newline at end of file diff --git a/src/Misc/contentHash/externals/linux-x64 b/src/Misc/contentHash/externals/linux-x64 deleted file mode 100644 index d23948a6888..00000000000 --- a/src/Misc/contentHash/externals/linux-x64 +++ /dev/null @@ -1 +0,0 @@ -50538de29f173bb73f708c4ed2c8328a62b8795829b97b2a6cb57197e2305287 \ No newline at end of file diff --git a/src/Misc/contentHash/externals/osx-arm64 b/src/Misc/contentHash/externals/osx-arm64 deleted file mode 100644 index bea235cd7fc..00000000000 --- a/src/Misc/contentHash/externals/osx-arm64 +++ /dev/null @@ -1 +0,0 @@ -a0a96cbb7593643b69e669bf14d7b29b7f27800b3a00bb3305aebe041456c701 \ No newline at end of file diff --git a/src/Misc/contentHash/externals/osx-x64 b/src/Misc/contentHash/externals/osx-x64 deleted file mode 100644 index d61ff6fd5b6..00000000000 --- a/src/Misc/contentHash/externals/osx-x64 +++ /dev/null @@ -1 +0,0 @@ -6255b22692779467047ecebd60ad46984866d75cdfe10421d593a7b51d620b09 \ No newline at end of file diff --git a/src/Misc/contentHash/externals/win-arm64 b/src/Misc/contentHash/externals/win-arm64 deleted file mode 100644 index d0bd205e541..00000000000 --- a/src/Misc/contentHash/externals/win-arm64 +++ /dev/null @@ -1 +0,0 @@ -6ff1abd055dc35bfbf06f75c2f08908f660346f66ad1d8f81c910068e9ba029d \ No newline at end of file diff --git a/src/Misc/contentHash/externals/win-x64 b/src/Misc/contentHash/externals/win-x64 deleted file mode 100644 index 1c8dd6223d2..00000000000 --- a/src/Misc/contentHash/externals/win-x64 +++ /dev/null @@ -1 +0,0 @@ -433a6d748742d12abd20dc2a79b62ac3d9718ae47ef26f8e84dc8c180eea3659 \ No newline at end of file From 76dc3a28c05a1d9fd52b3242a81cf3e5ae107e09 Mon Sep 17 00:00:00 2001 From: Patrick Ellis <319655+pje@users.noreply.github.com> Date: Tue, 14 May 2024 13:50:33 -0400 Subject: [PATCH 26/68] =?UTF-8?q?Upgrade=20node20:=2020.8.1=20=E2=86=92=20?= =?UTF-8?q?20.13.1=20(#3284)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Upgrade node20: 20.8.1 → 20.13.1 * Call out the release process for `alpine_nodejs` in a comment * move the comment to the end of the line so it's more obvious which variable it's talking about --- src/Misc/externals.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index 383221e4452..e057ecb218f 100755 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -5,10 +5,11 @@ PRECACHE=$2 NODE_URL=https://nodejs.org/dist UNOFFICIAL_NODE_URL=https://unofficial-builds.nodejs.org/download/release NODE_ALPINE_URL=https://github.com/actions/alpine_nodejs/releases/download +# When you update Node versions you must also create a new release of alpine_nodejs at that updated version. +# Follow the instructions here: https://github.com/actions/alpine_nodejs?tab=readme-ov-file#getting-started NODE16_VERSION="16.20.2" -NODE20_VERSION="20.8.1" -# used only for win-arm64, remove node16 unofficial version when official version is available -NODE16_UNOFFICIAL_VERSION="16.20.0" +NODE20_VERSION="20.13.1" +NODE16_UNOFFICIAL_VERSION="16.20.0" # used only for win-arm64, remove node16 unofficial version when official version is available get_abs_path() { # exploits the fact that pwd will print abs path when no args From 0f15173045445b004b3bc83845286b9b6f751e17 Mon Sep 17 00:00:00 2001 From: John Wesley Walker III <81404201+jww3@users.noreply.github.com> Date: Wed, 15 May 2024 20:01:06 +0200 Subject: [PATCH 27/68] Make it easy to install `git` on an Action Runner Image (#3273) (We don't actually install `git`. We simply get the prerequisites out of the way.) --- images/Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/images/Dockerfile b/images/Dockerfile index 2bd1a338473..97cd8118f9f 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -39,12 +39,14 @@ ENV RUNNER_MANUALLY_TRAP_SIG=1 ENV ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT=1 ENV ImageOS=ubuntu22 -RUN apt-get update -y \ - && apt-get install -y --no-install-recommends \ - sudo \ - lsb-release \ +RUN apt update -y \ + && apt install -y --no-install-recommends sudo lsb-release software-properties-common \ && rm -rf /var/lib/apt/lists/* +# Configure git-core/ppa based on guidance here: https://git-scm.com/download/linux +RUN add-apt-repository ppa:git-core/ppa \ + && apt update -y + RUN adduser --disabled-password --gecos "" --uid 1001 runner \ && groupadd docker --gid 123 \ && usermod -aG sudo runner \ From bd7235ef62c6fecfb76215ebe54b8b63fb30bbc9 Mon Sep 17 00:00:00 2001 From: John Wesley Walker III <81404201+jww3@users.noreply.github.com> Date: Fri, 17 May 2024 15:55:44 +0200 Subject: [PATCH 28/68] Install `gpg-agent` during actions/runner container image build (#3294) `add-apt-repository` depends on `gpg-agent` --- images/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/Dockerfile b/images/Dockerfile index 97cd8118f9f..81f5b9a288e 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -39,8 +39,9 @@ ENV RUNNER_MANUALLY_TRAP_SIG=1 ENV ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT=1 ENV ImageOS=ubuntu22 +# 'gpg-agent' and 'software-properties-common' are needed for the 'add-apt-repository' command that follows RUN apt update -y \ - && apt install -y --no-install-recommends sudo lsb-release software-properties-common \ + && apt install -y --no-install-recommends sudo lsb-release gpg-agent software-properties-common \ && rm -rf /var/lib/apt/lists/* # Configure git-core/ppa based on guidance here: https://git-scm.com/download/linux From ce4d7be00f4834b6889a82ea55020bb15fa834bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 10:47:43 -0400 Subject: [PATCH 29/68] Bump xunit from 2.4.1 to 2.7.1 in /src (#3242) * Bump xunit from 2.4.1 to 2.7.1 in /src Bumps [xunit](https://github.com/xunit/xunit) from 2.4.1 to 2.7.1. - [Commits](https://github.com/xunit/xunit/compare/2.4.1...2.7.1) --- updated-dependencies: - dependency-name: xunit dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Appease xunit warnings after upgrading to v2.7.1 * Appease the whitespace linter * Appease the whitespace linter --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Patrick Ellis <319655+pje@users.noreply.github.com> --- src/Test/L0/CommandLineParserL0.cs | 6 +++--- src/Test/L0/ConstantGenerationL0.cs | 2 +- src/Test/L0/Listener/CommandSettingsL0.cs | 6 +++--- .../Configuration/ConfigurationManagerL0.cs | 20 +++++++++---------- src/Test/L0/Listener/JobDispatcherL0.cs | 5 ++++- src/Test/L0/Listener/RunnerL0.cs | 19 +++++++++++++----- src/Test/L0/ProcessExtensionL0.cs | 2 +- src/Test/L0/RunnerWebProxyL0.cs | 18 +++++++++++++++-- src/Test/L0/Worker/ActionManagerL0.cs | 8 ++++---- src/Test/Test.csproj | 2 +- 10 files changed, 57 insertions(+), 31 deletions(-) diff --git a/src/Test/L0/CommandLineParserL0.cs b/src/Test/L0/CommandLineParserL0.cs index 19ab497fa35..e502868ba3b 100644 --- a/src/Test/L0/CommandLineParserL0.cs +++ b/src/Test/L0/CommandLineParserL0.cs @@ -68,7 +68,7 @@ public void ParsesCommands() trace.Info("Parsed"); trace.Info("Commands: {0}", clp.Commands.Count); - Assert.True(clp.Commands.Count == 2); + Assert.Equal(2, clp.Commands.Count); } } @@ -88,7 +88,7 @@ public void ParsesArgs() trace.Info("Parsed"); trace.Info("Args: {0}", clp.Args.Count); - Assert.True(clp.Args.Count == 2); + Assert.Equal(2, clp.Args.Count); Assert.True(clp.Args.ContainsKey("arg1")); Assert.Equal("arg1val", clp.Args["arg1"]); Assert.True(clp.Args.ContainsKey("arg2")); @@ -112,7 +112,7 @@ public void ParsesFlags() trace.Info("Parsed"); trace.Info("Args: {0}", clp.Flags.Count); - Assert.True(clp.Flags.Count == 2); + Assert.Equal(2, clp.Flags.Count); Assert.Contains("flag1", clp.Flags); Assert.Contains("flag2", clp.Flags); } diff --git a/src/Test/L0/ConstantGenerationL0.cs b/src/Test/L0/ConstantGenerationL0.cs index 204248516d7..f3c1b8f9eaf 100644 --- a/src/Test/L0/ConstantGenerationL0.cs +++ b/src/Test/L0/ConstantGenerationL0.cs @@ -24,7 +24,7 @@ public void BuildConstantGenerateSucceed() "osx-arm64" }; - Assert.True(BuildConstants.Source.CommitHash.Length == 40, $"CommitHash should be SHA-1 hash {BuildConstants.Source.CommitHash}"); + Assert.Equal(40, BuildConstants.Source.CommitHash.Length); Assert.True(validPackageNames.Contains(BuildConstants.RunnerPackage.PackageName), $"PackageName should be one of the following '{string.Join(", ", validPackageNames)}', current PackageName is '{BuildConstants.RunnerPackage.PackageName}'"); } } diff --git a/src/Test/L0/Listener/CommandSettingsL0.cs b/src/Test/L0/Listener/CommandSettingsL0.cs index ed7b672b86c..f823ba82f47 100644 --- a/src/Test/L0/Listener/CommandSettingsL0.cs +++ b/src/Test/L0/Listener/CommandSettingsL0.cs @@ -806,7 +806,7 @@ public void ValidateGoodCommandline() "test runner" }); // Assert. - Assert.True(command.Validate().Count == 0); + Assert.Equal(0, command.Validate().Count); } } @@ -844,7 +844,7 @@ public void ValidateGoodFlagCommandCombination(string validCommand, string flag) var command = new CommandSettings(hc, args: new string[] { validCommand, $"--{flag}" }); // Assert. - Assert.True(command.Validate().Count == 0); + Assert.Equal(0, command.Validate().Count); } } @@ -874,7 +874,7 @@ public void ValidateGoodArgCommandCombination(string validCommand, string arg, s var command = new CommandSettings(hc, args: new string[] { validCommand, $"--{arg}", argValue }); // Assert. - Assert.True(command.Validate().Count == 0); + Assert.Equal(0, command.Validate().Count); } } diff --git a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs index 5ee14404a61..3c698fdda12 100644 --- a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs +++ b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs @@ -190,11 +190,11 @@ public async Task CanEnsureConfigure() trace.Info("Configured, verifying all the parameter value"); var s = configManager.LoadSettings(); Assert.NotNull(s); - Assert.True(s.ServerUrl.Equals(_expectedServerUrl)); - Assert.True(s.AgentName.Equals(_expectedAgentName)); - Assert.True(s.PoolId.Equals(_secondRunnerGroupId)); - Assert.True(s.WorkFolder.Equals(_expectedWorkFolder)); - Assert.True(s.Ephemeral.Equals(true)); + Assert.Equal(_expectedServerUrl, s.ServerUrl); + Assert.Equal(_expectedAgentName, s.AgentName); + Assert.Equal(_secondRunnerGroupId, s.PoolId); + Assert.Equal(_expectedWorkFolder, s.WorkFolder); + Assert.True(s.Ephemeral); // validate GetAgentPoolsAsync gets called twice with automation pool type _runnerServer.Verify(x => x.GetAgentPoolsAsync(It.IsAny(), It.Is(p => p == TaskAgentPoolType.Automation)), Times.Exactly(2)); @@ -292,11 +292,11 @@ public async Task ConfigureDefaultLabelsDisabledWithCustomLabels() trace.Info("Configured, verifying all the parameter value"); var s = configManager.LoadSettings(); Assert.NotNull(s); - Assert.True(s.ServerUrl.Equals(_expectedServerUrl)); - Assert.True(s.AgentName.Equals(_expectedAgentName)); - Assert.True(s.PoolId.Equals(_secondRunnerGroupId)); - Assert.True(s.WorkFolder.Equals(_expectedWorkFolder)); - Assert.True(s.Ephemeral.Equals(true)); + Assert.Equal(_expectedServerUrl, s.ServerUrl); + Assert.Equal(_expectedAgentName, s.AgentName); + Assert.Equal(_secondRunnerGroupId, s.PoolId); + Assert.Equal(_expectedWorkFolder, s.WorkFolder); + Assert.True(s.Ephemeral); // validate GetAgentPoolsAsync gets called twice with automation pool type _runnerServer.Verify(x => x.GetAgentPoolsAsync(It.IsAny(), It.Is(p => p == TaskAgentPoolType.Automation)), Times.Exactly(2)); diff --git a/src/Test/L0/Listener/JobDispatcherL0.cs b/src/Test/L0/Listener/JobDispatcherL0.cs index 4d3f258c86c..cc50c180456 100644 --- a/src/Test/L0/Listener/JobDispatcherL0.cs +++ b/src/Test/L0/Listener/JobDispatcherL0.cs @@ -734,7 +734,10 @@ public async void DispatchesOneTimeJobRequest() await jobDispatcher.WaitAsync(CancellationToken.None); Assert.True(jobDispatcher.RunOnceJobCompleted.Task.IsCompleted, "JobDispatcher should set task complete token for one time agent."); - Assert.True(jobDispatcher.RunOnceJobCompleted.Task.Result, "JobDispatcher should set task complete token to 'TRUE' for one time agent."); + if (jobDispatcher.RunOnceJobCompleted.Task.IsCompleted) + { + Assert.True(await jobDispatcher.RunOnceJobCompleted.Task, "JobDispatcher should set task complete token to 'TRUE' for one time agent."); + } } } diff --git a/src/Test/L0/Listener/RunnerL0.cs b/src/Test/L0/Listener/RunnerL0.cs index 10b6d8c0fed..03f251a7720 100644 --- a/src/Test/L0/Listener/RunnerL0.cs +++ b/src/Test/L0/Listener/RunnerL0.cs @@ -126,7 +126,7 @@ public async void TestRunAsync() //wait for the runner to run one job if (!await signalWorkerComplete.WaitAsync(2000)) { - Assert.True(false, $"{nameof(_messageListener.Object.GetNextMessageAsync)} was not invoked."); + Assert.Fail($"{nameof(_messageListener.Object.GetNextMessageAsync)} was not invoked."); } else { @@ -305,8 +305,11 @@ public async void TestRunOnce() await Task.WhenAny(runnerTask, Task.Delay(30000)); Assert.True(runnerTask.IsCompleted, $"{nameof(runner.ExecuteCommand)} timed out."); - Assert.True(!runnerTask.IsFaulted, runnerTask.Exception?.ToString()); - Assert.True(runnerTask.Result == Constants.Runner.ReturnCode.Success); + Assert.False(runnerTask.IsFaulted, runnerTask.Exception?.ToString()); + if (runnerTask.IsCompleted) + { + Assert.Equal(Constants.Runner.ReturnCode.Success, await runnerTask); + } _jobDispatcher.Verify(x => x.Run(It.IsAny(), true), Times.Once(), $"{nameof(_jobDispatcher.Object.Run)} was not invoked."); @@ -406,7 +409,10 @@ public async void TestRunOnceOnlyTakeOneJobMessage() Assert.True(runnerTask.IsCompleted, $"{nameof(runner.ExecuteCommand)} timed out."); Assert.True(!runnerTask.IsFaulted, runnerTask.Exception?.ToString()); - Assert.True(runnerTask.Result == Constants.Runner.ReturnCode.Success); + if (runnerTask.IsCompleted) + { + Assert.Equal(Constants.Runner.ReturnCode.Success, await runnerTask); + } _jobDispatcher.Verify(x => x.Run(It.IsAny(), true), Times.Once(), $"{nameof(_jobDispatcher.Object.Run)} was not invoked."); @@ -492,7 +498,10 @@ public async void TestRunOnceHandleUpdateMessage() Assert.True(runnerTask.IsCompleted, $"{nameof(runner.ExecuteCommand)} timed out."); Assert.True(!runnerTask.IsFaulted, runnerTask.Exception?.ToString()); - Assert.True(runnerTask.Result == Constants.Runner.ReturnCode.RunOnceRunnerUpdating); + if (runnerTask.IsCompleted) + { + Assert.Equal(Constants.Runner.ReturnCode.RunOnceRunnerUpdating, await runnerTask); + } _updater.Verify(x => x.SelfUpdate(It.IsAny(), It.IsAny(), false, It.IsAny()), Times.Once); _jobDispatcher.Verify(x => x.Run(It.IsAny(), true), Times.Never()); diff --git a/src/Test/L0/ProcessExtensionL0.cs b/src/Test/L0/ProcessExtensionL0.cs index 9708c14953a..e9791250b20 100644 --- a/src/Test/L0/ProcessExtensionL0.cs +++ b/src/Test/L0/ProcessExtensionL0.cs @@ -58,7 +58,7 @@ public async Task SuccessReadProcessEnv() trace.Error(ex); } - Assert.True(false, "Fail to retrive process environment variable."); + Assert.Fail("Failed to retrieve process environment variable."); } finally { diff --git a/src/Test/L0/RunnerWebProxyL0.cs b/src/Test/L0/RunnerWebProxyL0.cs index 61fe68d181c..5e339e0a32f 100644 --- a/src/Test/L0/RunnerWebProxyL0.cs +++ b/src/Test/L0/RunnerWebProxyL0.cs @@ -65,7 +65,14 @@ public void IsNotUseRawHttpClientHandler() } } - Assert.True(badCode.Count == 0, $"The following code is using Raw HttpClientHandler() which will not follow the proxy setting agent have. Please use HostContext.CreateHttpClientHandler() instead.\n {string.Join("\n", badCode)}"); + if (badCode.Count > 0) + { + Assert.Fail($"The following code is using Raw HttpClientHandler() which will not follow the proxy setting agent have. Please use HostContext.CreateHttpClientHandler() instead.\n {string.Join("\n", badCode)}"); + } + else + { + Assert.True(true); + } } [Fact] @@ -112,7 +119,14 @@ public void IsNotUseRawHttpClient() } } - Assert.True(badCode.Count == 0, $"The following code is using Raw HttpClient() which will not follow the proxy setting agent have. Please use New HttpClient(HostContext.CreateHttpClientHandler()) instead.\n {string.Join("\n", badCode)}"); + if (badCode.Count > 0) + { + Assert.Fail($"The following code is using Raw HttpClient() which will not follow the proxy setting agent have. Please use New HttpClient(HostContext.CreateHttpClientHandler()) instead.\n {string.Join("\n", badCode)}"); + } + else + { + Assert.True(true); + } } [Fact] diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 1176966149f..91f183ae220 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -460,7 +460,7 @@ public async void PrepareActions_SkipDownloadActionForSelfRepo() //Act var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; - Assert.True(steps.Count == 0); + Assert.Equal(0, steps.Count); } finally { @@ -915,7 +915,7 @@ public async void PrepareActions_RepositoryActionWithActionfile_Node() var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; // node.js based action doesn't need any extra steps to build/pull containers. - Assert.True(steps.Count == 0); + Assert.Equal(0, steps.Count); } finally { @@ -1051,7 +1051,7 @@ public async void PrepareActions_CompositeActionWithActionfile_Node() var steps = (await _actionManager.PrepareActionsAsync(_ec.Object, actions)).ContainerSetupSteps; // node.js based action doesn't need any extra steps to build/pull containers. - Assert.True(steps.Count == 0); + Assert.Equal(0, steps.Count); var watermarkFile = Path.Combine(_hc.GetDirectory(WellKnownDirectory.Actions), "TingluoHuang/runner_L0", "CompositeBasic.completed"); Assert.True(File.Exists(watermarkFile)); // Comes from the composite action @@ -1245,7 +1245,7 @@ public void LoadsScriptActionDefinition() // Assert. Assert.NotNull(definition); Assert.NotNull(definition.Data); - Assert.True(definition.Data.Execution.ExecutionType == ActionExecutionType.Script); + Assert.Equal(ActionExecutionType.Script, definition.Data.Execution.ExecutionType); } finally { diff --git a/src/Test/Test.csproj b/src/Test/Test.csproj index 0f336013050..1beddbfc251 100644 --- a/src/Test/Test.csproj +++ b/src/Test/Test.csproj @@ -16,7 +16,7 @@ - + From 84b1bea43e5d87db3fec314caf9e49d4dedb77da Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Thu, 30 May 2024 13:36:44 +0100 Subject: [PATCH 30/68] Prepare relese 2.317.0 (#3311) --- releaseNote.md | 8 ++++++-- src/runnerversion | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 067b48bd66c..5033486a5e8 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,8 +1,12 @@ ## What's Changed -- Preserve dates when deserializing job message from Run Service by @ericsciple in https://github.com/actions/runner/pull/3269 +- Do not give up when uploading steps metadata by @yacaovsnc in https://github.com/actions/runner/pull/3280 +- Upgrade node20 to 20.13.1 by @pje in https://github.com/actions/runner/pull/3284 +- Delete all the contentHash files by @pje in https://github.com/actions/runner/pull/3285 +- Make it easy to install `git` on an Action Runner Image by @jww3 in https://github.com/actions/runner/pull/3273 +- Install `gpg-agent` during actions/runner container image build by @jww3 in https://github.com/actions/runner/pull/3294 -**Full Changelog**: https://github.com/actions/runner/compare/v2.316.0...v2.316.1 +**Full Changelog**: https://github.com/actions/runner/compare/v2.316.1...v2.317.0 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. diff --git a/src/runnerversion b/src/runnerversion index ad3450ba108..f7103c080f6 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.316.1 +2.317.0 From 00888c10f9331e08064e422d884490308e319b07 Mon Sep 17 00:00:00 2001 From: Hidetake Iwata Date: Sat, 1 Jun 2024 01:22:54 +0900 Subject: [PATCH 31/68] Bump docker version and docker buildx version (#3277) --- images/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/Dockerfile b/images/Dockerfile index 81f5b9a288e..707da217691 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -5,8 +5,8 @@ ARG TARGETOS ARG TARGETARCH ARG RUNNER_VERSION ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.0 -ARG DOCKER_VERSION=25.0.4 -ARG BUILDX_VERSION=0.13.1 +ARG DOCKER_VERSION=25.0.5 +ARG BUILDX_VERSION=0.13.2 RUN apt update -y && apt install curl unzip -y From edfdbb966118990c9898c6723121f5610188c155 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 4 Jun 2024 09:57:15 -0400 Subject: [PATCH 32/68] Make sure we mask secrets when reporting telemetry. (#3315) --- src/Runner.Common/JobServerQueue.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index dd060734b79..74c12bea28b 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -613,7 +613,7 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) private void SendResultsTelemetry(Exception ex) { - var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception with results. {ex.Message}" }; + var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception with results. {HostContext.SecretMasker.MaskSecrets(ex.Message)}" }; issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.ResultsUploadFailure; var telemetryRecord = new TimelineRecord() From 3f28dd845f5665426c4df81f2ef230d1cb1dd2da Mon Sep 17 00:00:00 2001 From: Josh Gross Date: Mon, 10 Jun 2024 18:13:17 -0400 Subject: [PATCH 33/68] Pass runner version as environment variable in workflow (#3318) --- .github/workflows/publish-image.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index fa906b29403..2932a268663 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -25,10 +25,12 @@ jobs: - name: Compute image version id: image uses: actions/github-script@v6 + env: + RUNNER_VERSION: ${{ github.event.inputs.runnerVersion }} with: script: | const fs = require('fs'); - const inputRunnerVersion = "${{ github.event.inputs.runnerVersion }}" + const inputRunnerVersion = process.env.RUNNER_VERSION; if (inputRunnerVersion) { console.log(`Using input runner version ${inputRunnerVersion}`) core.setOutput('version', inputRunnerVersion); From 1e74a8137b0f56bcabda735c43bd7caf1f602159 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 18 Jun 2024 11:28:53 -0400 Subject: [PATCH 34/68] Bump runner to dotnet 8 (#3345) * Bump runner to dotnet 8 * . --- .devcontainer/devcontainer.json | 2 +- .gitignore | 1 + src/Runner.Common/Runner.Common.csproj | 12 ++++++------ .../Configuration/RSAEncryptedFileKeyManager.cs | 2 ++ src/Runner.Listener/Runner.Listener.csproj | 13 +++++++------ src/Runner.Listener/Runner.cs | 2 ++ src/Runner.PluginHost/Runner.PluginHost.csproj | 5 +++-- src/Runner.Plugins/Runner.Plugins.csproj | 5 +++-- src/Runner.Sdk/Runner.Sdk.csproj | 11 ++++++----- src/Runner.Worker/Runner.Worker.csproj | 11 ++++++----- .../Common/Exceptions/PropertyExceptions.cs | 1 + src/Sdk/Common/Common/VssException.cs | 1 + src/Sdk/Sdk.csproj | 17 +++++++++-------- .../WebApi/WebApi/OAuth/VssOAuthExceptions.cs | 1 + .../WebApi/VssServiceResponseException.cs | 1 + src/Test/Test.csproj | 12 ++++++------ src/dev.sh | 2 +- src/global.json | 2 +- 18 files changed, 58 insertions(+), 43 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 032387bec99..c2069c7ba51 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,7 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:1": {}, "ghcr.io/devcontainers/features/dotnet": { - "version": "6.0.421" + "version": "8.0.204" }, "ghcr.io/devcontainers/features/node:1": { "version": "16" diff --git a/.gitignore b/.gitignore index 34d18c4cedc..411fe4011a5 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ _dotnetsdk TestResults TestLogs .DS_Store +.mono **/*.DotSettings.user \ No newline at end of file diff --git a/src/Runner.Common/Runner.Common.csproj b/src/Runner.Common/Runner.Common.csproj index 329a024aac6..6c4635626a2 100644 --- a/src/Runner.Common/Runner.Common.csproj +++ b/src/Runner.Common/Runner.Common.csproj @@ -1,11 +1,11 @@ - net6.0 + net8.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) @@ -15,11 +15,11 @@ - + - - - + + + diff --git a/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs b/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs index 15291be4387..a404a674e96 100644 --- a/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs +++ b/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs @@ -1,4 +1,5 @@ #if OS_WINDOWS +#pragma warning disable CA1416 using System.IO; using System.Security.Cryptography; using System.Text; @@ -84,4 +85,5 @@ void IRunnerService.Initialize(IHostContext context) } } } +#pragma warning restore CA1416 #endif diff --git a/src/Runner.Listener/Runner.Listener.csproj b/src/Runner.Listener/Runner.Listener.csproj index 3cd72ec61f8..afd528128a5 100644 --- a/src/Runner.Listener/Runner.Listener.csproj +++ b/src/Runner.Listener/Runner.Listener.csproj @@ -1,11 +1,12 @@ - net6.0 + net8.0 Exe win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) false true @@ -18,11 +19,11 @@ - + - - - + + + diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index 9acad839546..7509e112457 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -213,10 +213,12 @@ public async Task ExecuteCommand(CommandSettings command) var configFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), config.Key); var configContent = Convert.FromBase64String(config.Value); #if OS_WINDOWS +#pragma warning disable CA1416 if (configFile == HostContext.GetConfigFile(WellKnownConfigFile.RSACredentials)) { configContent = ProtectedData.Protect(configContent, null, DataProtectionScope.LocalMachine); } +#pragma warning restore CA1416 #endif File.WriteAllBytes(configFile, configContent); File.SetAttributes(configFile, File.GetAttributes(configFile) | FileAttributes.Hidden); diff --git a/src/Runner.PluginHost/Runner.PluginHost.csproj b/src/Runner.PluginHost/Runner.PluginHost.csproj index df30f3450a2..81a8d2e4304 100644 --- a/src/Runner.PluginHost/Runner.PluginHost.csproj +++ b/src/Runner.PluginHost/Runner.PluginHost.csproj @@ -1,11 +1,12 @@  - net6.0 + net8.0 Exe win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) false true diff --git a/src/Runner.Plugins/Runner.Plugins.csproj b/src/Runner.Plugins/Runner.Plugins.csproj index 39245a3f7a7..a786cf1cd1b 100644 --- a/src/Runner.Plugins/Runner.Plugins.csproj +++ b/src/Runner.Plugins/Runner.Plugins.csproj @@ -1,11 +1,12 @@  - net6.0 + net8.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) diff --git a/src/Runner.Sdk/Runner.Sdk.csproj b/src/Runner.Sdk/Runner.Sdk.csproj index 202e8669a64..55dbf12627c 100644 --- a/src/Runner.Sdk/Runner.Sdk.csproj +++ b/src/Runner.Sdk/Runner.Sdk.csproj @@ -1,11 +1,12 @@  - net6.0 + net8.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) @@ -14,9 +15,9 @@ - - - + + + diff --git a/src/Runner.Worker/Runner.Worker.csproj b/src/Runner.Worker/Runner.Worker.csproj index eee59b8721c..53c1610df3e 100644 --- a/src/Runner.Worker/Runner.Worker.csproj +++ b/src/Runner.Worker/Runner.Worker.csproj @@ -1,11 +1,12 @@ - net6.0 + net8.0 Exe win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) false true @@ -18,9 +19,9 @@ - - - + + + diff --git a/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs b/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs index 34c97b73a1d..72367654bf1 100644 --- a/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs +++ b/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs @@ -34,6 +34,7 @@ protected VssPropertyValidationException(SerializationInfo info, StreamingContex public String PropertyName { get; set; } + [Obsolete] [SecurityCritical] public override void GetObjectData(SerializationInfo info, StreamingContext context) { diff --git a/src/Sdk/Common/Common/VssException.cs b/src/Sdk/Common/Common/VssException.cs index 7cead78655d..5f8fb8c2554 100644 --- a/src/Sdk/Common/Common/VssException.cs +++ b/src/Sdk/Common/Common/VssException.cs @@ -127,6 +127,7 @@ protected VssException(SerializationInfo info, StreamingContext context) EventId = (int)info.GetValue("m_eventId", typeof(int)); } + [Obsolete] [SecurityCritical] public override void GetObjectData(SerializationInfo info, StreamingContext context) { diff --git a/src/Sdk/Sdk.csproj b/src/Sdk/Sdk.csproj index ff1cb85a4fe..a1ccd9be851 100644 --- a/src/Sdk/Sdk.csproj +++ b/src/Sdk/Sdk.csproj @@ -1,11 +1,12 @@ - net6.0 + net8.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true - NU1701;NU1603 + NU1701;NU1603;SYSLIB0050;SYSLIB0051 $(Version) TRACE 8.0 @@ -14,13 +15,13 @@ - + - - - - - + + + + + diff --git a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs index 5ebf86f9a8d..34ec103d05f 100644 --- a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs +++ b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs @@ -85,6 +85,7 @@ public String Error set; } + [Obsolete] public override void GetObjectData(SerializationInfo info, StreamingContext context) { base.GetObjectData(info, context); diff --git a/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs b/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs index e4aa84a736d..8dc275c0098 100644 --- a/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs +++ b/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs @@ -24,6 +24,7 @@ protected VssServiceResponseException(SerializationInfo info, StreamingContext c HttpStatusCode = (HttpStatusCode)info.GetInt32("HttpStatusCode"); } + [Obsolete] [SecurityCritical] public override void GetObjectData(SerializationInfo info, StreamingContext context) { diff --git a/src/Test/Test.csproj b/src/Test/Test.csproj index 1beddbfc251..aebe242096f 100644 --- a/src/Test/Test.csproj +++ b/src/Test/Test.csproj @@ -1,9 +1,9 @@ - net6.0 + net8.0 win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true - NU1701;NU1603;NU1603;xUnit2013; + NU1701;NU1603;NU1603;xUnit2013;SYSLIB0050;SYSLIB0051 @@ -15,13 +15,13 @@ - + - + - + - + diff --git a/src/dev.sh b/src/dev.sh index 0fc58761e53..29a5ecf1247 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -17,7 +17,7 @@ LAYOUT_DIR="$SCRIPT_DIR/../_layout" DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x" PACKAGE_DIR="$SCRIPT_DIR/../_package" DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk" -DOTNETSDK_VERSION="6.0.421" +DOTNETSDK_VERSION="8.0.204" DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION" RUNNER_VERSION=$(cat runnerversion) diff --git a/src/global.json b/src/global.json index e7028fe0dd4..1658e45125b 100644 --- a/src/global.json +++ b/src/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "6.0.421" + "version": "8.0.204" } } From 8f1c723ba0eeee0579cd6206c4f5b33b0a0a9931 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 18 Jun 2024 12:03:44 -0400 Subject: [PATCH 35/68] Upgrade dotnet sdk to v8.0.302 (#3346) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .devcontainer/devcontainer.json | 2 +- src/dev.sh | 2 +- src/global.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c2069c7ba51..75c7e3e7e64 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,7 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:1": {}, "ghcr.io/devcontainers/features/dotnet": { - "version": "8.0.204" + "version": "8.0.302" }, "ghcr.io/devcontainers/features/node:1": { "version": "16" diff --git a/src/dev.sh b/src/dev.sh index 29a5ecf1247..8ad3ae3505a 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -17,7 +17,7 @@ LAYOUT_DIR="$SCRIPT_DIR/../_layout" DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x" PACKAGE_DIR="$SCRIPT_DIR/../_package" DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk" -DOTNETSDK_VERSION="8.0.204" +DOTNETSDK_VERSION="8.0.302" DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION" RUNNER_VERSION=$(cat runnerversion) diff --git a/src/global.json b/src/global.json index 1658e45125b..c9c2078a746 100644 --- a/src/global.json +++ b/src/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "8.0.204" + "version": "8.0.302" } } From 3dab1f1fb072b4676aeb44c106fb05fa23efe13d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jun 2024 12:12:10 -0400 Subject: [PATCH 36/68] Bump System.Security.Cryptography.Pkcs from 5.0.0 to 8.0.0 in /src (#3347) Bumps [System.Security.Cryptography.Pkcs](https://github.com/dotnet/runtime) from 5.0.0 to 8.0.0. - [Release notes](https://github.com/dotnet/runtime/releases) - [Commits](https://github.com/dotnet/runtime/compare/v5.0.0...v8.0.0) --- updated-dependencies: - dependency-name: System.Security.Cryptography.Pkcs dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- src/Sdk/Sdk.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Sdk/Sdk.csproj b/src/Sdk/Sdk.csproj index a1ccd9be851..7ba739498a4 100644 --- a/src/Sdk/Sdk.csproj +++ b/src/Sdk/Sdk.csproj @@ -20,7 +20,7 @@ - + From ecb732eaf45ab06a08a4334a241b0baef3561a57 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Wed, 19 Jun 2024 11:38:32 -0500 Subject: [PATCH 37/68] Receive error body from Run Service (#3342) --- src/Sdk/DTWebApi/WebApi/Exceptions.cs | 20 ++++ src/Sdk/RSWebApi/Contracts/RunServiceError.cs | 17 +++ src/Sdk/RSWebApi/RunServiceHttpClient.cs | 110 ++++++++++++++++-- src/Sdk/WebApi/WebApi/RawHttpClientBase.cs | 61 +++++++++- src/Sdk/WebApi/WebApi/RawHttpClientResult.cs | 22 +++- 5 files changed, 211 insertions(+), 19 deletions(-) create mode 100644 src/Sdk/RSWebApi/Contracts/RunServiceError.cs diff --git a/src/Sdk/DTWebApi/WebApi/Exceptions.cs b/src/Sdk/DTWebApi/WebApi/Exceptions.cs index 536bf755055..ee47f137063 100644 --- a/src/Sdk/DTWebApi/WebApi/Exceptions.cs +++ b/src/Sdk/DTWebApi/WebApi/Exceptions.cs @@ -1539,6 +1539,26 @@ private TaskOrchestrationJobAlreadyAcquiredException(SerializationInfo info, Str } } + [Serializable] + [ExceptionMapping("0.0", "3.0", "TaskOrchestrationJobUnprocessableException", "GitHub.DistributedTask.WebApi.TaskOrchestrationJobUnprocessableException, GitHub.DistributedTask.WebApi, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a")] + public sealed class TaskOrchestrationJobUnprocessableException : DistributedTaskException + { + public TaskOrchestrationJobUnprocessableException(String message) + : base(message) + { + } + + public TaskOrchestrationJobUnprocessableException(String message, Exception innerException) + : base(message, innerException) + { + } + + private TaskOrchestrationJobUnprocessableException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } + [Serializable] [ExceptionMapping("0.0", "3.0", "TaskOrchestrationPlanSecurityException", "GitHub.DistributedTask.WebApi.TaskOrchestrationPlanSecurityException, GitHub.DistributedTask.WebApi, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a")] public sealed class TaskOrchestrationPlanSecurityException : DistributedTaskException diff --git a/src/Sdk/RSWebApi/Contracts/RunServiceError.cs b/src/Sdk/RSWebApi/Contracts/RunServiceError.cs new file mode 100644 index 00000000000..009a5914a06 --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/RunServiceError.cs @@ -0,0 +1,17 @@ +using System.Runtime.Serialization; + +namespace GitHub.Actions.RunService.WebApi +{ + [DataContract] + public class RunServiceError + { + [DataMember(Name = "source", EmitDefaultValue = false)] + public string Source { get; set; } + + [DataMember(Name = "statusCode", EmitDefaultValue = false)] + public int Code { get; set; } + + [DataMember(Name = "errorMessage", EmitDefaultValue = false)] + public string Message { get; set; } + } +} diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index ba176ccf6b6..14bdd2a6379 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -86,6 +86,7 @@ public async Task GetJobMessageAsync( httpMethod, requestUri: requestUri, content: requestContent, + readErrorBody: true, cancellationToken: cancellationToken); if (result.IsSuccess) @@ -93,14 +94,35 @@ public async Task GetJobMessageAsync( return result.Value; } + if (TryParseErrorBody(result.ErrorBody, out RunServiceError error)) + { + switch ((HttpStatusCode)error.Code) + { + case HttpStatusCode.NotFound: + throw new TaskOrchestrationJobNotFoundException($"Job message not found '{messageId}'. {error.Message}"); + case HttpStatusCode.Conflict: + throw new TaskOrchestrationJobAlreadyAcquiredException($"Job message already acquired '{messageId}'. {error.Message}"); + case HttpStatusCode.UnprocessableEntity: + throw new TaskOrchestrationJobUnprocessableException($"Unprocessable job '{messageId}'. {error.Message}"); + } + } + + // Temporary back compat switch (result.StatusCode) { case HttpStatusCode.NotFound: throw new TaskOrchestrationJobNotFoundException($"Job message not found: {messageId}"); case HttpStatusCode.Conflict: throw new TaskOrchestrationJobAlreadyAcquiredException($"Job message already acquired: {messageId}"); - default: - throw new Exception($"Failed to get job message: {result.Error}"); + } + + if (!string.IsNullOrEmpty(result.ErrorBody)) + { + throw new Exception($"Failed to get job message: {result.Error}. {Truncate(result.ErrorBody)}"); + } + else + { + throw new Exception($"Failed to get job message: {result.Error}"); } } @@ -108,7 +130,7 @@ public async Task CompleteJobAsync( Uri requestUri, Guid planId, Guid jobId, - TaskResult result, + TaskResult conclusion, Dictionary outputs, IList stepResults, IList jobAnnotations, @@ -120,7 +142,7 @@ public async Task CompleteJobAsync( { PlanID = planId, JobID = jobId, - Conclusion = result, + Conclusion = conclusion, Outputs = outputs, StepResults = stepResults, Annotations = jobAnnotations, @@ -130,22 +152,39 @@ public async Task CompleteJobAsync( requestUri = new Uri(requestUri, "completejob"); var requestContent = new ObjectContent(payload, new VssJsonMediaTypeFormatter(true)); - var response = await SendAsync( + var result = await Send2Async( httpMethod, requestUri, content: requestContent, cancellationToken: cancellationToken); - if (response.IsSuccessStatusCode) + if (result.IsSuccess) { return; } - switch (response.StatusCode) + if (TryParseErrorBody(result.ErrorBody, out RunServiceError error)) + { + switch ((HttpStatusCode)error.Code) + { + case HttpStatusCode.NotFound: + throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}. {error.Message}"); + } + } + + // Temporary back compat + switch (result.StatusCode) { case HttpStatusCode.NotFound: throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}"); - default: - throw new Exception($"Failed to complete job: {response.ReasonPhrase}"); + } + + if (!string.IsNullOrEmpty(result.ErrorBody)) + { + throw new Exception($"Failed to complete job: {result.Error}. {Truncate(result.ErrorBody)}"); + } + else + { + throw new Exception($"Failed to complete job: {result.Error}"); } } @@ -169,6 +208,7 @@ public async Task RenewJobAsync( httpMethod, requestUri, content: requestContent, + readErrorBody: true, cancellationToken: cancellationToken); if (result.IsSuccess) @@ -176,12 +216,29 @@ public async Task RenewJobAsync( return result.Value; } + if (TryParseErrorBody(result.ErrorBody, out RunServiceError error)) + { + switch ((HttpStatusCode)error.Code) + { + case HttpStatusCode.NotFound: + throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}. {error.Message}"); + } + } + + // Temporary back compat switch (result.StatusCode) { case HttpStatusCode.NotFound: throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}"); - default: - throw new Exception($"Failed to renew job: {result.Error}"); + } + + if (!string.IsNullOrEmpty(result.ErrorBody)) + { + throw new Exception($"Failed to renew job: {result.Error}. {Truncate(result.ErrorBody)}"); + } + else + { + throw new Exception($"Failed to renew job: {result.Error}"); } } @@ -190,5 +247,36 @@ public async Task RenewJobAsync( var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); return JsonConvert.DeserializeObject(json, s_serializerSettings); } + + private static bool TryParseErrorBody(string errorBody, out RunServiceError error) + { + if (!string.IsNullOrEmpty(errorBody)) + { + try + { + error = JsonUtility.FromString(errorBody); + if (error?.Source == "actions-run-service") + { + return true; + } + } + catch (Exception) + { + } + } + + error = null; + return false; + } + + private static string Truncate(string errorBody) + { + if (errorBody.Length > 100) + { + return errorBody.Substring(0, 100) + "[truncated]"; + } + + return errorBody; + } } } diff --git a/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs b/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs index de7c3bcb372..23c51472487 100644 --- a/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs +++ b/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs @@ -101,15 +101,55 @@ protected async Task SendAsync( } } + protected async Task Send2Async( + HttpMethod method, + Uri requestUri, + HttpContent content = null, + IEnumerable> queryParameters = null, + Object userState = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + using (var response = await SendAsync(method, requestUri, content, queryParameters, userState, cancellationToken).ConfigureAwait(false)) + { + if (response.IsSuccessStatusCode) + { + return new RawHttpClientResult( + isSuccess: true, + error: string.Empty, + statusCode: response.StatusCode); + } + else + { + var errorBody = default(string); + try + { + errorBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + errorBody = $"Error reading HTTP response body: {ex.Message}"; + } + + string errorMessage = $"Error: {response.ReasonPhrase}"; + return new RawHttpClientResult( + isSuccess: false, + error: errorMessage, + statusCode: response.StatusCode, + errorBody: errorBody); + } + } + } + protected Task> SendAsync( HttpMethod method, Uri requestUri, HttpContent content = null, IEnumerable> queryParameters = null, + Boolean readErrorBody = false, Object userState = null, CancellationToken cancellationToken = default(CancellationToken)) { - return SendAsync(method, null, requestUri, content, queryParameters, userState, cancellationToken); + return SendAsync(method, null, requestUri, content, queryParameters, readErrorBody, userState, cancellationToken); } protected async Task> SendAsync( @@ -118,18 +158,20 @@ protected async Task> SendAsync( Uri requestUri, HttpContent content = null, IEnumerable> queryParameters = null, + Boolean readErrorBody = false, Object userState = null, CancellationToken cancellationToken = default(CancellationToken)) { using (VssTraceActivity.GetOrCreate().EnterCorrelationScope()) using (HttpRequestMessage requestMessage = CreateRequestMessage(method, additionalHeaders, requestUri, content, queryParameters)) { - return await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false); + return await SendAsync(requestMessage, readErrorBody, userState, cancellationToken).ConfigureAwait(false); } } protected async Task> SendAsync( HttpRequestMessage message, + Boolean readErrorBody = false, Object userState = null, CancellationToken cancellationToken = default(CancellationToken)) { @@ -145,8 +187,21 @@ protected async Task> SendAsync( } else { + var errorBody = default(string); + if (readErrorBody) + { + try + { + errorBody = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + errorBody = $"Error reading HTTP response body: {ex.Message}"; + } + } + string errorMessage = $"Error: {response.ReasonPhrase}"; - return RawHttpClientResult.Fail(errorMessage, response.StatusCode); + return RawHttpClientResult.Fail(errorMessage, response.StatusCode, errorBody); } } } diff --git a/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs b/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs index 1b2dc5f06cc..113de871fe8 100644 --- a/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs +++ b/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs @@ -5,15 +5,27 @@ namespace Sdk.WebApi.WebApi public class RawHttpClientResult { public bool IsSuccess { get; protected set; } + + /// + /// A description of the HTTP status code, like "Error: Unprocessable Entity" + /// public string Error { get; protected set; } + + /// + /// The HTTP response body for unsuccessful HTTP status codes, or an error message when reading the response body fails. + /// + public string ErrorBody { get; protected set; } + public HttpStatusCode StatusCode { get; protected set; } + public bool IsFailure => !IsSuccess; - protected RawHttpClientResult(bool isSuccess, string error, HttpStatusCode statusCode) + public RawHttpClientResult(bool isSuccess, string error, HttpStatusCode statusCode, string errorBody = null) { IsSuccess = isSuccess; Error = error; StatusCode = statusCode; + ErrorBody = errorBody; } } @@ -21,13 +33,13 @@ public class RawHttpClientResult : RawHttpClientResult { public T Value { get; private set; } - protected internal RawHttpClientResult(T value, bool isSuccess, string error, HttpStatusCode statusCode) - : base(isSuccess, error, statusCode) + protected internal RawHttpClientResult(T value, bool isSuccess, string error, HttpStatusCode statusCode, string errorBody) + : base(isSuccess, error, statusCode, errorBody) { Value = value; } - public static RawHttpClientResult Fail(string message, HttpStatusCode statusCode) => new RawHttpClientResult(default(T), false, message, statusCode); - public static RawHttpClientResult Ok(T value) => new RawHttpClientResult(value, true, string.Empty, HttpStatusCode.OK); + public static RawHttpClientResult Fail(string message, HttpStatusCode statusCode, string errorBody) => new RawHttpClientResult(default(T), false, message, statusCode, errorBody); + public static RawHttpClientResult Ok(T value) => new RawHttpClientResult(value, true, string.Empty, HttpStatusCode.OK, null); } } From 054fc2e0463b4640a0e9fcaae97aea2279bd81b0 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Mon, 24 Jun 2024 16:33:22 -0500 Subject: [PATCH 38/68] Backoff to avoid excessive retries to Run Service in a duration (#3354) --- src/Runner.Common/RunServer.cs | 5 +- src/Runner.Listener/ErrorThrottler.cs | 44 +++++ src/Runner.Listener/Runner.cs | 28 ++- src/Test/L0/Listener/ErrorThrottlerL0.cs | 213 +++++++++++++++++++++++ src/Test/L0/Listener/RunnerL0.cs | 9 + src/Test/L0/TestHostContext.cs | 25 +++ 6 files changed, 318 insertions(+), 6 deletions(-) create mode 100644 src/Runner.Listener/ErrorThrottler.cs create mode 100644 src/Test/L0/Listener/ErrorThrottlerL0.cs diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index c042796b124..50ad0556018 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -62,7 +62,10 @@ public Task GetJobMessageAsync(string id, CancellationTo CheckConnection(); return RetryRequest( async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, VarUtil.OS, cancellationToken), cancellationToken, - shouldRetry: ex => ex is not TaskOrchestrationJobAlreadyAcquiredException); + shouldRetry: ex => + ex is not TaskOrchestrationJobNotFoundException && // HTTP status 404 + ex is not TaskOrchestrationJobAlreadyAcquiredException && // HTTP status 409 + ex is not TaskOrchestrationJobUnprocessableException); // HTTP status 422 } public Task CompleteJobAsync( diff --git a/src/Runner.Listener/ErrorThrottler.cs b/src/Runner.Listener/ErrorThrottler.cs new file mode 100644 index 00000000000..8525c728573 --- /dev/null +++ b/src/Runner.Listener/ErrorThrottler.cs @@ -0,0 +1,44 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Runner.Common; +using GitHub.Services.Common; + +namespace GitHub.Runner.Listener +{ + [ServiceLocator(Default = typeof(ErrorThrottler))] + public interface IErrorThrottler : IRunnerService + { + void Reset(); + Task IncrementAndWaitAsync(CancellationToken token); + } + + public sealed class ErrorThrottler : RunnerService, IErrorThrottler + { + internal static readonly TimeSpan MinBackoff = TimeSpan.FromSeconds(1); + internal static readonly TimeSpan MaxBackoff = TimeSpan.FromMinutes(1); + internal static readonly TimeSpan BackoffCoefficient = TimeSpan.FromSeconds(1); + private int _count = 0; + + public void Reset() + { + _count = 0; + } + + public async Task IncrementAndWaitAsync(CancellationToken token) + { + if (++_count <= 1) + { + return; + } + + TimeSpan backoff = BackoffTimerHelper.GetExponentialBackoff( + attempt: _count - 2, // 0-based attempt + minBackoff: MinBackoff, + maxBackoff: MaxBackoff, + deltaBackoff: BackoffCoefficient); + Trace.Warning($"Back off {backoff.TotalSeconds} seconds before next attempt. Current consecutive error count: {_count}"); + await HostContext.Delay(backoff, token); + } + } +} diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index 7509e112457..d83cefa0645 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -32,10 +32,25 @@ public sealed class Runner : RunnerService, IRunner private bool _inConfigStage; private ManualResetEvent _completedCommand = new(false); + // + // Helps avoid excessive calls to Run Service when encountering non-retriable errors from /acquirejob. + // Normally we rely on the HTTP clients to back off between retry attempts. However, acquiring a job + // involves calls to both Run Serivce and Broker. And Run Service and Broker communicate with each other + // in an async fashion. + // + // When Run Service encounters a non-retriable error, it sends an async message to Broker. The runner will, + // however, immediately call Broker to get the next message. If the async event from Run Service to Broker + // has not yet been processed, the next message from Broker may be the same job message. + // + // The error throttler helps us back off when encountering successive, non-retriable errors from /acquirejob. + // + private IErrorThrottler _acquireJobThrottler; + public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); _term = HostContext.GetService(); + _acquireJobThrottler = HostContext.CreateService(); } public async Task ExecuteCommand(CommandSettings command) @@ -565,13 +580,16 @@ private async Task RunAsync(RunnerSettings settings, bool runOnce = false) await runServer.ConnectAsync(new Uri(messageRef.RunServiceUrl), creds); try { - jobRequestMessage = - await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, - messageQueueLoopTokenSource.Token); + jobRequestMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token); + _acquireJobThrottler.Reset(); } - catch (TaskOrchestrationJobAlreadyAcquiredException) + catch (Exception ex) when ( + ex is TaskOrchestrationJobNotFoundException || // HTTP status 404 + ex is TaskOrchestrationJobAlreadyAcquiredException || // HTTP status 409 + ex is TaskOrchestrationJobUnprocessableException) // HTTP status 422 { - Trace.Info("Job is already acquired, skip this message."); + Trace.Info($"Skipping message Job. {ex.Message}"); + await _acquireJobThrottler.IncrementAndWaitAsync(messageQueueLoopTokenSource.Token); continue; } catch (Exception ex) diff --git a/src/Test/L0/Listener/ErrorThrottlerL0.cs b/src/Test/L0/Listener/ErrorThrottlerL0.cs new file mode 100644 index 00000000000..e4118b181f0 --- /dev/null +++ b/src/Test/L0/Listener/ErrorThrottlerL0.cs @@ -0,0 +1,213 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Listener; +using GitHub.Runner.Listener.Configuration; +using GitHub.Runner.Common.Tests; +using System.Runtime.CompilerServices; +using GitHub.Services.WebApi; +using Moq; +using Xunit; + +namespace GitHub.Runner.Common.Tests.Listener +{ + public sealed class ErrorThrottlerL0 + { + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(3)] + [InlineData(4)] + [InlineData(5)] + [InlineData(6)] + [InlineData(7)] + [InlineData(8)] + public async void TestIncrementAndWait(int totalAttempts) + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange + var errorThrottler = new ErrorThrottler(); + errorThrottler.Initialize(hc); + var eventArgs = new List(); + hc.Delaying += (sender, args) => + { + eventArgs.Add(args); + }; + + // Act + for (int attempt = 1; attempt <= totalAttempts; attempt++) + { + await errorThrottler.IncrementAndWaitAsync(CancellationToken.None); + } + + // Assert + Assert.Equal(totalAttempts - 1, eventArgs.Count); + for (int i = 0; i < eventArgs.Count; i++) + { + // Expected milliseconds + int expectedMin; + int expectedMax; + + switch (i) + { + case 0: + expectedMin = 1000; // Min backoff + expectedMax = 1000; + break; + case 1: + expectedMin = 1800; // Min + 0.8 * Coefficient + expectedMax = 2200; // Min + 1.2 * Coefficient + break; + case 2: + expectedMin = 3400; // Min + 0.8 * Coefficient * 3 + expectedMax = 4600; // Min + 1.2 * Coefficient * 3 + break; + case 3: + expectedMin = 6600; // Min + 0.8 * Coefficient * 7 + expectedMax = 9400; // Min + 1.2 * Coefficient * 7 + break; + case 4: + expectedMin = 13000; // Min + 0.8 * Coefficient * 15 + expectedMax = 19000; // Min + 1.2 * Coefficient * 15 + break; + case 5: + expectedMin = 25800; // Min + 0.8 * Coefficient * 31 + expectedMax = 38200; // Min + 1.2 * Coefficient * 31 + break; + case 6: + expectedMin = 51400; // Min + 0.8 * Coefficient * 63 + expectedMax = 60000; // Max backoff + break; + case 7: + expectedMin = 60000; + expectedMax = 60000; + break; + default: + throw new NotSupportedException("Unexpected eventArgs count"); + } + + var actualMilliseconds = eventArgs[i].Delay.TotalMilliseconds; + Assert.True(expectedMin <= actualMilliseconds, $"Unexpected min delay for eventArgs[{i}]. Expected min {expectedMin}, actual {actualMilliseconds}"); + Assert.True(expectedMax >= actualMilliseconds, $"Unexpected max delay for eventArgs[{i}]. Expected max {expectedMax}, actual {actualMilliseconds}"); + } + } + } + + [Fact] + public async void TestReset() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange + var errorThrottler = new ErrorThrottler(); + errorThrottler.Initialize(hc); + var eventArgs = new List(); + hc.Delaying += (sender, args) => + { + eventArgs.Add(args); + }; + + // Act + await errorThrottler.IncrementAndWaitAsync(CancellationToken.None); + await errorThrottler.IncrementAndWaitAsync(CancellationToken.None); + await errorThrottler.IncrementAndWaitAsync(CancellationToken.None); + errorThrottler.Reset(); + await errorThrottler.IncrementAndWaitAsync(CancellationToken.None); + await errorThrottler.IncrementAndWaitAsync(CancellationToken.None); + await errorThrottler.IncrementAndWaitAsync(CancellationToken.None); + + // Assert + Assert.Equal(4, eventArgs.Count); + for (int i = 0; i < eventArgs.Count; i++) + { + // Expected milliseconds + int expectedMin; + int expectedMax; + + switch (i) + { + case 0: + case 2: + expectedMin = 1000; // Min backoff + expectedMax = 1000; + break; + case 1: + case 3: + expectedMin = 1800; // Min + 0.8 * Coefficient + expectedMax = 2200; // Min + 1.2 * Coefficient + break; + default: + throw new NotSupportedException("Unexpected eventArgs count"); + } + + var actualMilliseconds = eventArgs[i].Delay.TotalMilliseconds; + Assert.True(expectedMin <= actualMilliseconds, $"Unexpected min delay for eventArgs[{i}]. Expected min {expectedMin}, actual {actualMilliseconds}"); + Assert.True(expectedMax >= actualMilliseconds, $"Unexpected max delay for eventArgs[{i}]. Expected max {expectedMax}, actual {actualMilliseconds}"); + } + } + } + + [Fact] + public async void TestReceivesCancellationToken() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange + var errorThrottler = new ErrorThrottler(); + errorThrottler.Initialize(hc); + var eventArgs = new List(); + hc.Delaying += (sender, args) => + { + eventArgs.Add(args); + }; + var cancellationTokenSource1 = new CancellationTokenSource(); + var cancellationTokenSource2 = new CancellationTokenSource(); + var cancellationTokenSource3 = new CancellationTokenSource(); + + // Act + await errorThrottler.IncrementAndWaitAsync(cancellationTokenSource1.Token); + await errorThrottler.IncrementAndWaitAsync(cancellationTokenSource2.Token); + await errorThrottler.IncrementAndWaitAsync(cancellationTokenSource3.Token); + + // Assert + Assert.Equal(2, eventArgs.Count); + Assert.Equal(cancellationTokenSource2.Token, eventArgs[0].Token); + Assert.Equal(cancellationTokenSource3.Token, eventArgs[1].Token); + } + } + + [Fact] + public async void TestReceivesSender() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange + var errorThrottler = new ErrorThrottler(); + errorThrottler.Initialize(hc); + var senders = new List(); + hc.Delaying += (sender, args) => + { + senders.Add(sender); + }; + + // Act + await errorThrottler.IncrementAndWaitAsync(CancellationToken.None); + await errorThrottler.IncrementAndWaitAsync(CancellationToken.None); + await errorThrottler.IncrementAndWaitAsync(CancellationToken.None); + + // Assert + Assert.Equal(2, senders.Count); + Assert.Equal(hc, senders[0]); + Assert.Equal(hc, senders[1]); + } + } + + private TestHostContext CreateTestContext([CallerMemberName] String testName = "") + { + return new TestHostContext(this, testName); + } + } +} diff --git a/src/Test/L0/Listener/RunnerL0.cs b/src/Test/L0/Listener/RunnerL0.cs index 03f251a7720..b29f8835c2b 100644 --- a/src/Test/L0/Listener/RunnerL0.cs +++ b/src/Test/L0/Listener/RunnerL0.cs @@ -23,6 +23,7 @@ public sealed class RunnerL0 private Mock _term; private Mock _configStore; private Mock _updater; + private Mock _acquireJobThrottler; public RunnerL0() { @@ -35,6 +36,7 @@ public RunnerL0() _term = new Mock(); _configStore = new Mock(); _updater = new Mock(); + _acquireJobThrottler = new Mock(); } private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName) @@ -67,6 +69,7 @@ public async void TestRunAsync() hc.SetSingleton(_promptManager.Object); hc.SetSingleton(_runnerServer.Object); hc.SetSingleton(_configStore.Object); + hc.EnqueueInstance(_acquireJobThrottler.Object); runner.Initialize(hc); var settings = new RunnerSettings { @@ -174,6 +177,7 @@ public async void TestExecuteCommandForRunAsService(string[] args, bool configur hc.SetSingleton(_promptManager.Object); hc.SetSingleton(_messageListener.Object); hc.SetSingleton(_configStore.Object); + hc.EnqueueInstance(_acquireJobThrottler.Object); var command = new CommandSettings(hc, args); @@ -205,6 +209,7 @@ public async void TestMachineProvisionerCLI() hc.SetSingleton(_promptManager.Object); hc.SetSingleton(_messageListener.Object); hc.SetSingleton(_configStore.Object); + hc.EnqueueInstance(_acquireJobThrottler.Object); var command = new CommandSettings(hc, new[] { "run" }); @@ -242,6 +247,7 @@ public async void TestRunOnce() hc.SetSingleton(_promptManager.Object); hc.SetSingleton(_runnerServer.Object); hc.SetSingleton(_configStore.Object); + hc.EnqueueInstance(_acquireJobThrottler.Object); runner.Initialize(hc); var settings = new RunnerSettings { @@ -338,6 +344,7 @@ public async void TestRunOnceOnlyTakeOneJobMessage() hc.SetSingleton(_promptManager.Object); hc.SetSingleton(_runnerServer.Object); hc.SetSingleton(_configStore.Object); + hc.EnqueueInstance(_acquireJobThrottler.Object); runner.Initialize(hc); var settings = new RunnerSettings { @@ -439,6 +446,7 @@ public async void TestRunOnceHandleUpdateMessage() hc.SetSingleton(_runnerServer.Object); hc.SetSingleton(_configStore.Object); hc.SetSingleton(_updater.Object); + hc.EnqueueInstance(_acquireJobThrottler.Object); runner.Initialize(hc); var settings = new RunnerSettings @@ -522,6 +530,7 @@ public async void TestRemoveLocalRunnerConfig() hc.SetSingleton(_configurationManager.Object); hc.SetSingleton(_configStore.Object); hc.SetSingleton(_promptManager.Object); + hc.EnqueueInstance(_acquireJobThrottler.Object); var command = new CommandSettings(hc, new[] { "remove", "--local" }); diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index a3e484b14c8..c44f13f1c4a 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -30,9 +30,11 @@ public sealed class TestHostContext : IHostContext, IDisposable private string _tempDirectoryRoot = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("D")); private StartupType _startupType; public event EventHandler Unloading; + public event EventHandler Delaying; public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token; public ShutdownReason RunnerShutdownReason { get; private set; } public ISecretMasker SecretMasker => _secretMasker; + public TestHostContext(object testClass, [CallerMemberName] string testName = "") { ArgUtil.NotNull(testClass, nameof(testClass)); @@ -92,6 +94,14 @@ public StartupType StartupType public async Task Delay(TimeSpan delay, CancellationToken token) { + // Event callback + EventHandler handler = Delaying; + if (handler != null) + { + handler(this, new DelayEventArgs(delay, token)); + } + + // Delay zero await Task.Delay(TimeSpan.Zero); } @@ -361,4 +371,19 @@ private void LoadContext_Unloading(AssemblyLoadContext obj) } } } + + public class DelayEventArgs : EventArgs + { + public DelayEventArgs( + TimeSpan delay, + CancellationToken token) + { + Delay = delay; + Token = token; + } + + public TimeSpan Delay { get; } + + public CancellationToken Token { get; } + } } From 70746ff593636b07ad251a1525a3fabd1a7a36e9 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Wed, 26 Jun 2024 14:56:33 +0200 Subject: [PATCH 39/68] Bump hook version to 0.6.1 (#3350) --- images/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/Dockerfile b/images/Dockerfile index 707da217691..39f1ab05674 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -4,7 +4,7 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0-jammy as build ARG TARGETOS ARG TARGETARCH ARG RUNNER_VERSION -ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.0 +ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.1 ARG DOCKER_VERSION=25.0.5 ARG BUILDX_VERSION=0.13.2 From 8df87a82b01bd4ea5768ccd2442ad666d7b4b77d Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Mon, 8 Jul 2024 12:41:06 +0100 Subject: [PATCH 40/68] Rephrase node20 warning (#3376) --- src/Runner.Common/Constants.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 9378104c435..d68d5cdf695 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -181,7 +181,7 @@ public static class Features public static readonly string DeprecatedNodeVersion = "node16"; public static readonly string EnforcedNode12DetectedAfterEndOfLife = "The following actions uses node12 which is deprecated and will be forced to run on node16: {0}. For more info: https://github.blog/changelog/2023-06-13-github-actions-all-actions-will-run-on-node16-instead-of-node12-by-default/"; public static readonly string EnforcedNode12DetectedAfterEndOfLifeEnvVariable = "Node16ForceActionsWarnings"; - public static readonly string EnforcedNode16DetectedAfterEndOfLife = "The following actions uses Node.js version which is deprecated and will be forced to run on node20: {0}. For more info: https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/"; + public static readonly string EnforcedNode16DetectedAfterEndOfLife = "The following actions use a deprecated Node.js version and will be forced to run on node20: {0}. For more info: https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/"; public static readonly string EnforcedNode16DetectedAfterEndOfLifeEnvVariable = "Node20ForceActionsWarnings"; } From dbcaa7cf3d893cb8a801b381f86718d4efb3e053 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 01:06:31 -0400 Subject: [PATCH 41/68] Upgrade dotnet sdk to v8.0.303 (#3388) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .devcontainer/devcontainer.json | 2 +- src/dev.sh | 2 +- src/global.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 75c7e3e7e64..b4745fe10da 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,7 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:1": {}, "ghcr.io/devcontainers/features/dotnet": { - "version": "8.0.302" + "version": "8.0.303" }, "ghcr.io/devcontainers/features/node:1": { "version": "16" diff --git a/src/dev.sh b/src/dev.sh index 8ad3ae3505a..8120ef33267 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -17,7 +17,7 @@ LAYOUT_DIR="$SCRIPT_DIR/../_layout" DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x" PACKAGE_DIR="$SCRIPT_DIR/../_package" DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk" -DOTNETSDK_VERSION="8.0.302" +DOTNETSDK_VERSION="8.0.303" DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION" RUNNER_VERSION=$(cat runnerversion) diff --git a/src/global.json b/src/global.json index c9c2078a746..fd07d882ad3 100644 --- a/src/global.json +++ b/src/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "8.0.302" + "version": "8.0.303" } } From 2a7f327d93fb79326c974f4858ce62c3b81c580a Mon Sep 17 00:00:00 2001 From: Kynan Ware <47394200+BagToad@users.noreply.github.com> Date: Wed, 17 Jul 2024 08:35:03 -0600 Subject: [PATCH 42/68] Update supported distros to match new docs (#3226) Co-authored-by: Patrick Ellis <319655+pje@users.noreply.github.com> --- docs/start/envlinux.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/start/envlinux.md b/docs/start/envlinux.md index 11eff187693..c3a871f92bf 100644 --- a/docs/start/envlinux.md +++ b/docs/start/envlinux.md @@ -5,15 +5,15 @@ ## Supported Distributions and Versions x64 - - Red Hat Enterprise Linux 7+ - - CentOS 7+ - - Oracle Linux 7+ + - Red Hat Enterprise Linux 8+ + - CentOS 8+ + - Oracle Linux 8+ - Fedora 29+ - - Debian 9+ - - Ubuntu 16.04+ - - Linux Mint 18+ - - openSUSE 15+ - - SUSE Enterprise Linux (SLES) 12 SP2+ + - Debian 10+ + - Ubuntu 20.04+ + - Linux Mint 20+ + - openSUSE 15.2+ + - SUSE Enterprise Linux (SLES) 15 SP2+ ## Install .Net Core 3.x Linux Dependencies From 41bc0da6fe09466d23fd37d691feeb68dd3b4338 Mon Sep 17 00:00:00 2001 From: Kynan Ware <47394200+BagToad@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:07:09 -0600 Subject: [PATCH 43/68] Redirect supported OS doc section to the public docs (#3396) * redirect supported OS doc to public docs * Anchor to appropriate OS heading Co-authored-by: Patrick Ellis <319655+pje@users.noreply.github.com> --------- Co-authored-by: Patrick Ellis <319655+pje@users.noreply.github.com> --- docs/start/envlinux.md | 11 +---------- docs/start/envosx.md | 3 +-- docs/start/envwin.md | 7 +------ 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/docs/start/envlinux.md b/docs/start/envlinux.md index c3a871f92bf..f313debe37d 100644 --- a/docs/start/envlinux.md +++ b/docs/start/envlinux.md @@ -4,16 +4,7 @@ ## Supported Distributions and Versions -x64 - - Red Hat Enterprise Linux 8+ - - CentOS 8+ - - Oracle Linux 8+ - - Fedora 29+ - - Debian 10+ - - Ubuntu 20.04+ - - Linux Mint 20+ - - openSUSE 15.2+ - - SUSE Enterprise Linux (SLES) 15 SP2+ +Please see "[Supported architectures and operating systems for self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#linux)." ## Install .Net Core 3.x Linux Dependencies diff --git a/docs/start/envosx.md b/docs/start/envosx.md index 7a2e95fc85b..5bc0c87c958 100644 --- a/docs/start/envosx.md +++ b/docs/start/envosx.md @@ -4,7 +4,6 @@ ## Supported Versions - - macOS High Sierra (10.13) and later versions - - x64 and arm64 (Apple Silicon) +Please see "[Supported architectures and operating systems for self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#macos)." ## [More .Net Core Prerequisites Information](https://docs.microsoft.com/en-us/dotnet/core/macos-prerequisites?tabs=netcore30) diff --git a/docs/start/envwin.md b/docs/start/envwin.md index 76d392b8683..531c00cf849 100644 --- a/docs/start/envwin.md +++ b/docs/start/envwin.md @@ -2,11 +2,6 @@ ## Supported Versions - - Windows 7 64-bit - - Windows 8.1 64-bit - - Windows 10 64-bit - - Windows Server 2012 R2 64-bit - - Windows Server 2016 64-bit - - Windows Server 2019 64-bit +Please see "[Supported architectures and operating systems for self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#windows)." ## [More .NET Core Prerequisites Information](https://docs.microsoft.com/en-us/dotnet/core/windows-prerequisites?tabs=netcore30) From 2190396357dff9076091cb1f7d5781c207f41f33 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Fri, 26 Jul 2024 10:36:05 -0400 Subject: [PATCH 44/68] Update Docker to v27.1.1 (#3401) * Update Docker to v27.1.1 * Update Dockerfile --- images/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/Dockerfile b/images/Dockerfile index 39f1ab05674..dd8437e1d27 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -5,8 +5,8 @@ ARG TARGETOS ARG TARGETARCH ARG RUNNER_VERSION ARG RUNNER_CONTAINER_HOOKS_VERSION=0.6.1 -ARG DOCKER_VERSION=25.0.5 -ARG BUILDX_VERSION=0.13.2 +ARG DOCKER_VERSION=27.1.1 +ARG BUILDX_VERSION=0.16.2 RUN apt update -y && apt install curl unzip -y From 12506842c0f11159f0c0399774897b5b61a48433 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Fri, 26 Jul 2024 10:03:59 -0500 Subject: [PATCH 45/68] Prepare release 2.318.0 (#3404) --- releaseNote.md | 20 +++++++++++++------- src/runnerversion | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 5033486a5e8..5a2f71142f6 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,12 +1,18 @@ ## What's Changed -- Do not give up when uploading steps metadata by @yacaovsnc in https://github.com/actions/runner/pull/3280 -- Upgrade node20 to 20.13.1 by @pje in https://github.com/actions/runner/pull/3284 -- Delete all the contentHash files by @pje in https://github.com/actions/runner/pull/3285 -- Make it easy to install `git` on an Action Runner Image by @jww3 in https://github.com/actions/runner/pull/3273 -- Install `gpg-agent` during actions/runner container image build by @jww3 in https://github.com/actions/runner/pull/3294 - -**Full Changelog**: https://github.com/actions/runner/compare/v2.316.1...v2.317.0 +- Update Docker to v27.1.1 by @TingluoHuang in https://github.com/actions/runner/pull/3401 +- Upgrade dotnet sdk to v8.0.303 in https://github.com/actions/runner/pull/3388 +- Rephrase node20 warning by @rentziass in https://github.com/actions/runner/pull/3376 +- Bump hook version to 0.6.1 by @nikola-jokic in https://github.com/actions/runner/pull/3350 +- Backoff to avoid excessive retries to Run Service in a duration by @ericsciple in https://github.com/actions/runner/pull/3354 +- Bump System.Security.Cryptography.Pkcs from 5.0.0 to 8.0.0 in /src in https://github.com/actions/runner/pull/3347 +- Upgrade dotnet sdk to v8.0.302 in https://github.com/actions/runner/pull/3346 +- Bump runner to dotnet 8 by @TingluoHuang in https://github.com/actions/runner/pull/3345 +- Pass runner version as environment variable in workflow by @joshmgross in https://github.com/actions/runner/pull/3318 +- Make sure we mask secrets when reporting telemetry by @TingluoHuang in https://github.com/actions/runner/pull/3315 +- Bump docker version and docker buildx version by @int128 in https://github.com/actions/runner/pull/3277 + +**Full Changelog**: https://github.com/actions/runner/compare/v2.317.0...v2.318.0 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. diff --git a/src/runnerversion b/src/runnerversion index f7103c080f6..4d0a47ffd8e 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.317.0 +2.318.0 From ae04147f960f2232d0a888c89d3e128f92f21677 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Fri, 2 Aug 2024 14:37:46 -0500 Subject: [PATCH 46/68] Warn for soon to be deprecated OS versions (#3413) --- src/Runner.Worker/JobExtension.cs | 4 + src/Runner.Worker/OSWarningChecker.cs | 87 ++++++ .../Pipelines/AgentJobRequestMessage.cs | 26 ++ src/Sdk/DTPipelines/Pipelines/OSWarning.cs | 44 +++ src/Test/L0/Listener/JobDispatcherL0.cs | 3 +- src/Test/L0/Listener/RunnerL0.cs | 2 +- src/Test/L0/Worker/ActionCommandManagerL0.cs | 2 +- .../L0/Worker/CreateStepSummaryCommandL0.cs | 2 +- src/Test/L0/Worker/ExecutionContextL0.cs | 32 +-- src/Test/L0/Worker/JobExtensionL0.cs | 3 +- src/Test/L0/Worker/JobRunnerL0.cs | 1 + src/Test/L0/Worker/OSWarningCheckerL0.cs | 271 ++++++++++++++++++ src/Test/L0/Worker/WorkerL0.cs | 2 +- 13 files changed, 457 insertions(+), 22 deletions(-) create mode 100644 src/Runner.Worker/OSWarningChecker.cs create mode 100644 src/Sdk/DTPipelines/Pipelines/OSWarning.cs create mode 100644 src/Test/L0/Worker/OSWarningCheckerL0.cs diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index c420ddfc88f..4ca06b90c94 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -127,6 +127,10 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } } + // Check OS warning + var osWarningChecker = HostContext.GetService(); + await osWarningChecker.CheckOSAsync(context, message.OSWarnings); + try { var tokenPermissions = jobContext.Global.Variables.Get("system.github.token.permissions") ?? ""; diff --git a/src/Runner.Worker/OSWarningChecker.cs b/src/Runner.Worker/OSWarningChecker.cs new file mode 100644 index 00000000000..900f8f1f0eb --- /dev/null +++ b/src/Runner.Worker/OSWarningChecker.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using System.Text.RegularExpressions; +using GitHub.DistributedTask.WebApi; +using GitHub.DistributedTask.Pipelines; +using GitHub.Runner.Common; +using GitHub.Runner.Sdk; + +namespace GitHub.Runner.Worker +{ + [ServiceLocator(Default = typeof(OSWarningChecker))] + public interface IOSWarningChecker : IRunnerService + { + Task CheckOSAsync(IExecutionContext context, IList osWarnings); + } + +#if OS_WINDOWS || OS_OSX + public sealed class OSWarningChecker : RunnerService, IOSWarningChecker + { + public Task CheckOSAsync(IExecutionContext context, IList osWarnings) + { + ArgUtil.NotNull(context, nameof(context)); + ArgUtil.NotNull(osWarnings, nameof(osWarnings)); + return Task.CompletedTask; + } + } +#else + public sealed class OSWarningChecker : RunnerService, IOSWarningChecker + { + private static readonly TimeSpan s_matchTimeout = TimeSpan.FromMilliseconds(100); + private static readonly RegexOptions s_regexOptions = RegexOptions.CultureInvariant | RegexOptions.IgnoreCase; + + public async Task CheckOSAsync(IExecutionContext context, IList osWarnings) + { + ArgUtil.NotNull(context, nameof(context)); + ArgUtil.NotNull(osWarnings, nameof(osWarnings)); + foreach (var osWarning in osWarnings) + { + if (string.IsNullOrEmpty(osWarning.FilePath)) + { + Trace.Error("The file path is not specified in the OS warning check."); + continue; + } + + if (string.IsNullOrEmpty(osWarning.RegularExpression)) + { + Trace.Error("The regular expression is not specified in the OS warning check."); + continue; + } + + if (string.IsNullOrEmpty(osWarning.Warning)) + { + Trace.Error("The warning message is not specified in the OS warning check."); + continue; + } + + try + { + if (File.Exists(osWarning.FilePath)) + { + var lines = await File.ReadAllLinesAsync(osWarning.FilePath, context.CancellationToken); + var regex = new Regex(osWarning.RegularExpression, s_regexOptions, s_matchTimeout); + foreach (var line in lines) + { + if (regex.IsMatch(line)) + { + context.Warning(osWarning.Warning); + context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $"OS warning: {osWarning.Warning}" }); + return; + } + } + } + } + catch (Exception ex) + { + Trace.Error("An error occurred while checking OS warnings for file '{0}' and regex '{1}'.", osWarning.FilePath, osWarning.RegularExpression); + Trace.Error(ex); + context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $"An error occurred while checking OS warnings for file '{osWarning.FilePath}' and regex '{osWarning.RegularExpression}': {ex.Message}" }); + } + } + } + } +#endif +} + diff --git a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs index 070d86ee24f..030f1d82610 100644 --- a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs +++ b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs @@ -44,6 +44,7 @@ public AgentJobRequestMessage( IList defaults, ActionsEnvironmentReference actionsEnvironment, TemplateToken snapshot, + IList osWarnings, String messageType = JobRequestMessageTypes.PipelineAgentJobRequest) { this.MessageType = messageType; @@ -73,6 +74,11 @@ public AgentJobRequestMessage( m_defaults = new List(defaults); } + if (osWarnings?.Count > 0) + { + m_osWarnings = new List(osWarnings); + } + this.ContextData = new Dictionary(StringComparer.OrdinalIgnoreCase); if (contextData?.Count > 0) { @@ -288,6 +294,18 @@ public IList FileTable } } + public IList OSWarnings + { + get + { + if (m_osWarnings == null) + { + m_osWarnings = new List(); + } + return m_osWarnings; + } + } + // todo: remove after feature-flag DistributedTask.EvaluateContainerOnRunner is enabled everywhere public void SetJobSidecarContainers(IDictionary value) { @@ -425,6 +443,11 @@ private void OnSerializing(StreamingContext context) { JobContainer = new StringToken(null, null, null, m_jobContainerResourceAlias); } + + if (m_osWarnings?.Count == 0) + { + m_osWarnings = null; + } } [DataMember(Name = "EnvironmentVariables", EmitDefaultValue = false)] @@ -449,6 +472,9 @@ private void OnSerializing(StreamingContext context) [DataMember(Name = "JobSidecarContainers", EmitDefaultValue = false)] private IDictionary m_jobSidecarContainers; + [DataMember(Name = "OSWarnings", EmitDefaultValue = false)] + private List m_osWarnings; + // todo: remove after feature-flag DistributedTask.EvaluateContainerOnRunner is enabled everywhere [IgnoreDataMember] private string m_jobContainerResourceAlias; diff --git a/src/Sdk/DTPipelines/Pipelines/OSWarning.cs b/src/Sdk/DTPipelines/Pipelines/OSWarning.cs new file mode 100644 index 00000000000..7eb4168b26f --- /dev/null +++ b/src/Sdk/DTPipelines/Pipelines/OSWarning.cs @@ -0,0 +1,44 @@ +using System; +using System.ComponentModel; +using System.Runtime.Serialization; + +namespace GitHub.DistributedTask.Pipelines +{ + /// + /// Information to check whether the OS is going to be deprecated soon + /// + [DataContract] + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class OSWarning + { + /// + /// Gets or sets the file to check + /// + [DataMember(EmitDefaultValue = false)] + public String FilePath + { + get; + set; + } + + /// + /// Gets or sets the regular expression to match + /// + [DataMember(EmitDefaultValue = false)] + public String RegularExpression + { + get; + set; + } + + /// + /// Gets or sets the warning annotation message, if the regular expression matches the content of the file + /// + [DataMember(EmitDefaultValue = false)] + public String Warning + { + get; + set; + } + } +} diff --git a/src/Test/L0/Listener/JobDispatcherL0.cs b/src/Test/L0/Listener/JobDispatcherL0.cs index cc50c180456..c1d5346fe0a 100644 --- a/src/Test/L0/Listener/JobDispatcherL0.cs +++ b/src/Test/L0/Listener/JobDispatcherL0.cs @@ -41,7 +41,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage() TaskOrchestrationPlanReference plan = new(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); - var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); result.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); return result; } @@ -810,6 +810,7 @@ private static AgentJobRequestMessage GetAgentJobRequestMessage() null, new List(), new ActionsEnvironmentReference("env"), + null, null ); return message; diff --git a/src/Test/L0/Listener/RunnerL0.cs b/src/Test/L0/Listener/RunnerL0.cs index b29f8835c2b..3010dcb402a 100644 --- a/src/Test/L0/Listener/RunnerL0.cs +++ b/src/Test/L0/Listener/RunnerL0.cs @@ -44,7 +44,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName) TaskOrchestrationPlanReference plan = new(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); - return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); } private JobCancelMessage CreateJobCancelMessage() diff --git a/src/Test/L0/Worker/ActionCommandManagerL0.cs b/src/Test/L0/Worker/ActionCommandManagerL0.cs index 3a1f8f70fe9..8be95ae4da5 100644 --- a/src/Test/L0/Worker/ActionCommandManagerL0.cs +++ b/src/Test/L0/Worker/ActionCommandManagerL0.cs @@ -232,7 +232,7 @@ public void EchoProcessCommandDebugOn() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs b/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs index 185f44b3865..972d6843bd7 100644 --- a/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs +++ b/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs @@ -193,7 +193,7 @@ private TestHostContext Setup([CallerMemberName] string name = "") TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "Summary Job"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 08abcd09585..0054c969938 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -29,7 +29,7 @@ public void AddIssue_CountWarningsErrors() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -106,7 +106,7 @@ public void ApplyContinueOnError_CheckResultAndOutcome() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -162,7 +162,7 @@ public void AddIssue_TrimMessageSize() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -216,7 +216,7 @@ public void AddIssue_OverrideLogMessage() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -271,7 +271,7 @@ public void AddIssue_AddStepAndLineNumberInformation() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -322,7 +322,7 @@ public void Debug_Multilines() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -373,7 +373,7 @@ public void RegisterPostJobAction_ShareState() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -471,7 +471,7 @@ public void RegisterPostJobAction_NotRegisterPostTwice() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -555,7 +555,7 @@ public void ActionResult_Lowercase() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -610,7 +610,7 @@ public void PublishStepTelemetry_RegularStep_NoOpt() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -653,7 +653,7 @@ public void PublishStepTelemetry_RegularStep() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -717,7 +717,7 @@ public void PublishStepTelemetry_EmbeddedStep() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -781,7 +781,7 @@ public void PublishStepResult_EmbeddedStep() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -969,7 +969,7 @@ public void ActionVariables_AddedToVarsContext() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -1014,7 +1014,7 @@ public void ActionVariables_DebugUsingVars() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -1057,7 +1057,7 @@ public void ActionVariables_SecretsPrecedenceForDebugUsingVars() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index d66ded66372..6ada9143dc3 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -114,7 +114,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " }; Guid jobId = Guid.NewGuid(); - _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null, null); + _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null, null, null); GitHubContext github = new(); github["repository"] = new Pipelines.ContextData.StringContextData("actions/runner"); github["secret_source"] = new Pipelines.ContextData.StringContextData("Actions"); @@ -140,6 +140,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " hc.SetSingleton(_diagnosticLogManager.Object); hc.SetSingleton(_jobHookProvider.Object); hc.SetSingleton(_snapshotOperationProvider.Object); + hc.SetSingleton(new Mock().Object); hc.EnqueueInstance(_logger.Object); // JobExecutionContext hc.EnqueueInstance(_logger.Object); // job start hook hc.EnqueueInstance(_logger.Object); // Initial Job diff --git a/src/Test/L0/Worker/JobRunnerL0.cs b/src/Test/L0/Worker/JobRunnerL0.cs index e8011b9b051..c1021150c6f 100644 --- a/src/Test/L0/Worker/JobRunnerL0.cs +++ b/src/Test/L0/Worker/JobRunnerL0.cs @@ -102,6 +102,7 @@ private Pipelines.AgentJobRequestMessage GetMessage(String messageType = JobRequ testName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, new ActionsEnvironmentReference("staging"), null, + null, messageType: messageType); message.Variables[Constants.Variables.System.Culture] = "en-US"; message.Resources.Endpoints.Add(new ServiceEndpoint() diff --git a/src/Test/L0/Worker/OSWarningCheckerL0.cs b/src/Test/L0/Worker/OSWarningCheckerL0.cs new file mode 100644 index 00000000000..7f4d0f32ee2 --- /dev/null +++ b/src/Test/L0/Worker/OSWarningCheckerL0.cs @@ -0,0 +1,271 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using GitHub.DistributedTask.Pipelines; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Worker; +using Moq; +using Xunit; + +namespace GitHub.Runner.Common.Tests.Worker +{ + public sealed class OSWarningCheckerL0 + { + private CancellationTokenSource _ecTokenSource; + private Mock _ec; + private TestHostContext _hc; + private OSWarningChecker _osWarningChecker; + private List _issues; + private string _workFolder; + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void CheckOS_FileNotExists() + { + try + { + // Arrange + Setup(); + var osWarnings = new List + { + new OSWarning + { + FilePath = Path.Combine(_workFolder, "os-release"), + RegularExpression = "some OS version", + Warning = "Some OS version will be deprecated soon" + }, + }; + + // Act + await _osWarningChecker.CheckOSAsync(_ec.Object, osWarnings); + + // Assert + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _ec.Object.Global.JobTelemetry.Count); + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void CheckOS_CaseInsensitive() + { + try + { + // Arrange + Setup(); + var osWarnings = new List + { + new OSWarning + { + FilePath = Path.Combine(_workFolder, "os-release"), + RegularExpression = "some OS verSION", + Warning = "Some OS version will be deprecated soon" + }, + }; + File.WriteAllText(Path.Combine(_workFolder, "os-release"), "some OS version\n"); + + // Act + await _osWarningChecker.CheckOSAsync(_ec.Object, osWarnings); + +#if OS_WINDOWS || OS_OSX + // Assert + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _ec.Object.Global.JobTelemetry.Count); +#else + // Assert + Assert.Equal(1, _issues.Count); + Assert.Equal(IssueType.Warning, _issues[0].Type); + Assert.Equal("Some OS version will be deprecated soon", _issues[0].Message); + Assert.Equal(1, _ec.Object.Global.JobTelemetry.Count); + Assert.Equal(JobTelemetryType.General, _ec.Object.Global.JobTelemetry[0].Type); + Assert.Equal("OS warning: Some OS version will be deprecated soon", _ec.Object.Global.JobTelemetry[0].Message); +#endif + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void CheckOS_MatchesOnceWithinAFile() + { + try + { + // Arrange + Setup(); + var osWarnings = new List + { + new OSWarning + { + FilePath = Path.Combine(_workFolder, "os-release"), + RegularExpression = "some OS version", + Warning = "Some OS version will be deprecated soon" + }, + }; + File.WriteAllText(Path.Combine(_workFolder, "os-release"), "some OS version\nsome OS version\n"); + + // Act + await _osWarningChecker.CheckOSAsync(_ec.Object, osWarnings); + +#if OS_WINDOWS || OS_OSX + // Assert + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _ec.Object.Global.JobTelemetry.Count); +#else + // Assert + Assert.Equal(1, _issues.Count); + Assert.Equal(IssueType.Warning, _issues[0].Type); + Assert.Equal("Some OS version will be deprecated soon", _issues[0].Message); + Assert.Equal(1, _ec.Object.Global.JobTelemetry.Count); + Assert.Equal(JobTelemetryType.General, _ec.Object.Global.JobTelemetry[0].Type); + Assert.Equal("OS warning: Some OS version will be deprecated soon", _ec.Object.Global.JobTelemetry[0].Message); +#endif + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void CheckOS_MatchesOnceAcrossFiles() + { + try + { + // Arrange + Setup(); + var osWarnings = new List + { + new OSWarning + { + FilePath = Path.Combine(_workFolder, "os-release"), + RegularExpression = "some OS version", + Warning = "Some OS version will be deprecated soon" + }, + new OSWarning + { + FilePath = Path.Combine(_workFolder, "os-release-2"), + RegularExpression = "some OS version", + Warning = "Some OS version will be deprecated soon" + }, + }; + File.WriteAllText(Path.Combine(_workFolder, "os-release"), "some OS version\n"); + File.WriteAllText(Path.Combine(_workFolder, "os-release-2"), "some OS version\n"); + + // Act + await _osWarningChecker.CheckOSAsync(_ec.Object, osWarnings); + +#if OS_WINDOWS || OS_OSX + // Assert + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _ec.Object.Global.JobTelemetry.Count); +#else + // Assert + Assert.Equal(1, _issues.Count); + Assert.Equal(IssueType.Warning, _issues[0].Type); + Assert.Equal("Some OS version will be deprecated soon", _issues[0].Message); + Assert.Equal(1, _ec.Object.Global.JobTelemetry.Count); + Assert.Equal(JobTelemetryType.General, _ec.Object.Global.JobTelemetry[0].Type); + Assert.Equal("OS warning: Some OS version will be deprecated soon", _ec.Object.Global.JobTelemetry[0].Message); +#endif + } + finally + { + Teardown(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void CheckOS_LogsTelemetryUponException() + { + try + { + // Arrange + Setup(); + var osWarnings = new List + { + new OSWarning + { + FilePath = Path.Combine(_workFolder, "os-release"), + RegularExpression = "abc[", // Invalid pattern + Warning = "Some OS version will be deprecated soon" + }, + }; + File.WriteAllText(Path.Combine(_workFolder, "os-release"), "some OS version\n"); + + // Act + await _osWarningChecker.CheckOSAsync(_ec.Object, osWarnings); + +#if OS_WINDOWS || OS_OSX + // Assert + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _ec.Object.Global.JobTelemetry.Count); +#else + // Assert + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _ec.Object.Global.JobTelemetry.Count); + Assert.Equal(JobTelemetryType.General, _ec.Object.Global.JobTelemetry[0].Type); + Assert.Equal( + $"An error occurred while checking OS warnings for file '{osWarnings[0].FilePath}' and regex '{osWarnings[0].RegularExpression}': Invalid pattern 'abc[' at offset 4. Unterminated [] set.", + _ec.Object.Global.JobTelemetry[0].Message); +#endif + } + finally + { + Teardown(); + } + } + + private void Setup([CallerMemberName] string name = "") + { + _issues = new List(); + + // Test host context + _hc = new TestHostContext(this, name); + + // Random work folder + _workFolder = _hc.GetDirectory(WellKnownDirectory.Work); + Directory.CreateDirectory(_workFolder); + + // Execution context token source + _ecTokenSource?.Dispose(); + _ecTokenSource = new CancellationTokenSource(); + + // Execution context + _ec = new Mock(); + _ec.Setup(x => x.Global).Returns(new GlobalContext()); + _ec.Setup(x => x.CancellationToken).Returns(_ecTokenSource.Token); + _ec.Object.Global.JobTelemetry = new List(); + _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, ExecutionContextLogOptions logOptions) => { _issues.Add(issue); }); + + // OS warning checker + _osWarningChecker = new OSWarningChecker(); + _osWarningChecker.Initialize(_hc); + } + + private void Teardown() + { + _hc?.Dispose(); + if (!string.IsNullOrEmpty(_workFolder) && Directory.Exists(_workFolder)) + { + Directory.Delete(_workFolder, recursive: true); + } + } + } +} diff --git a/src/Test/L0/Worker/WorkerL0.cs b/src/Test/L0/Worker/WorkerL0.cs index defcc981404..ba6a933afdd 100644 --- a/src/Test/L0/Worker/WorkerL0.cs +++ b/src/Test/L0/Worker/WorkerL0.cs @@ -67,7 +67,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName) new Pipelines.ContextData.DictionaryContextData() }, }; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List(), resources, context, null, actions, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List(), resources, context, null, actions, null, null, null, null, null, null); return jobRequest; } From 43d67e46db3fedfd8247d8d6fb968a4e11765643 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 5 Aug 2024 11:03:18 -0400 Subject: [PATCH 47/68] Revert "Bump runner to dotnet 8" (#3412) * Revert "Upgrade dotnet sdk to v8.0.303 (#3388)" This reverts commit dbcaa7cf3d893cb8a801b381f86718d4efb3e053. * Revert "Bump System.Security.Cryptography.Pkcs from 5.0.0 to 8.0.0 in /src (#3347)" This reverts commit 3dab1f1fb072b4676aeb44c106fb05fa23efe13d. * Revert "Upgrade dotnet sdk to v8.0.302 (#3346)" This reverts commit 8f1c723ba0eeee0579cd6206c4f5b33b0a0a9931. * Revert "Bump runner to dotnet 8 (#3345)" This reverts commit 1e74a8137b0f56bcabda735c43bd7caf1f602159. --- .devcontainer/devcontainer.json | 2 +- .gitignore | 1 - src/Runner.Common/Runner.Common.csproj | 12 ++++++------ .../Configuration/RSAEncryptedFileKeyManager.cs | 2 -- src/Runner.Listener/Runner.Listener.csproj | 13 ++++++------- src/Runner.Listener/Runner.cs | 2 -- src/Runner.PluginHost/Runner.PluginHost.csproj | 5 ++--- src/Runner.Plugins/Runner.Plugins.csproj | 5 ++--- src/Runner.Sdk/Runner.Sdk.csproj | 11 +++++------ src/Runner.Worker/Runner.Worker.csproj | 11 +++++------ .../Common/Exceptions/PropertyExceptions.cs | 1 - src/Sdk/Common/Common/VssException.cs | 1 - src/Sdk/Sdk.csproj | 17 ++++++++--------- .../WebApi/WebApi/OAuth/VssOAuthExceptions.cs | 1 - .../WebApi/VssServiceResponseException.cs | 1 - src/Test/Test.csproj | 12 ++++++------ src/dev.sh | 2 +- src/global.json | 2 +- 18 files changed, 43 insertions(+), 58 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b4745fe10da..032387bec99 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,7 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:1": {}, "ghcr.io/devcontainers/features/dotnet": { - "version": "8.0.303" + "version": "6.0.421" }, "ghcr.io/devcontainers/features/node:1": { "version": "16" diff --git a/.gitignore b/.gitignore index 411fe4011a5..34d18c4cedc 100644 --- a/.gitignore +++ b/.gitignore @@ -26,5 +26,4 @@ _dotnetsdk TestResults TestLogs .DS_Store -.mono **/*.DotSettings.user \ No newline at end of file diff --git a/src/Runner.Common/Runner.Common.csproj b/src/Runner.Common/Runner.Common.csproj index 6c4635626a2..329a024aac6 100644 --- a/src/Runner.Common/Runner.Common.csproj +++ b/src/Runner.Common/Runner.Common.csproj @@ -1,11 +1,11 @@ - net8.0 + net6.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true - NU1701;NU1603;SYSLIB0050;SYSLIB0051 + NU1701;NU1603 $(Version) @@ -15,11 +15,11 @@ - + - - - + + + diff --git a/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs b/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs index a404a674e96..15291be4387 100644 --- a/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs +++ b/src/Runner.Listener/Configuration/RSAEncryptedFileKeyManager.cs @@ -1,5 +1,4 @@ #if OS_WINDOWS -#pragma warning disable CA1416 using System.IO; using System.Security.Cryptography; using System.Text; @@ -85,5 +84,4 @@ void IRunnerService.Initialize(IHostContext context) } } } -#pragma warning restore CA1416 #endif diff --git a/src/Runner.Listener/Runner.Listener.csproj b/src/Runner.Listener/Runner.Listener.csproj index afd528128a5..3cd72ec61f8 100644 --- a/src/Runner.Listener/Runner.Listener.csproj +++ b/src/Runner.Listener/Runner.Listener.csproj @@ -1,12 +1,11 @@ - net8.0 + net6.0 Exe win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 - true true - NU1701;NU1603;SYSLIB0050;SYSLIB0051 + NU1701;NU1603 $(Version) false true @@ -19,11 +18,11 @@ - + - - - + + + diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index d83cefa0645..e2f638522b8 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -228,12 +228,10 @@ public async Task ExecuteCommand(CommandSettings command) var configFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), config.Key); var configContent = Convert.FromBase64String(config.Value); #if OS_WINDOWS -#pragma warning disable CA1416 if (configFile == HostContext.GetConfigFile(WellKnownConfigFile.RSACredentials)) { configContent = ProtectedData.Protect(configContent, null, DataProtectionScope.LocalMachine); } -#pragma warning restore CA1416 #endif File.WriteAllBytes(configFile, configContent); File.SetAttributes(configFile, File.GetAttributes(configFile) | FileAttributes.Hidden); diff --git a/src/Runner.PluginHost/Runner.PluginHost.csproj b/src/Runner.PluginHost/Runner.PluginHost.csproj index 81a8d2e4304..df30f3450a2 100644 --- a/src/Runner.PluginHost/Runner.PluginHost.csproj +++ b/src/Runner.PluginHost/Runner.PluginHost.csproj @@ -1,12 +1,11 @@  - net8.0 + net6.0 Exe win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 - true true - NU1701;NU1603;SYSLIB0050;SYSLIB0051 + NU1701;NU1603 $(Version) false true diff --git a/src/Runner.Plugins/Runner.Plugins.csproj b/src/Runner.Plugins/Runner.Plugins.csproj index a786cf1cd1b..39245a3f7a7 100644 --- a/src/Runner.Plugins/Runner.Plugins.csproj +++ b/src/Runner.Plugins/Runner.Plugins.csproj @@ -1,12 +1,11 @@  - net8.0 + net6.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 - true true - NU1701;NU1603;SYSLIB0050;SYSLIB0051 + NU1701;NU1603 $(Version) diff --git a/src/Runner.Sdk/Runner.Sdk.csproj b/src/Runner.Sdk/Runner.Sdk.csproj index 55dbf12627c..202e8669a64 100644 --- a/src/Runner.Sdk/Runner.Sdk.csproj +++ b/src/Runner.Sdk/Runner.Sdk.csproj @@ -1,12 +1,11 @@  - net8.0 + net6.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 - true true - NU1701;NU1603;SYSLIB0050;SYSLIB0051 + NU1701;NU1603 $(Version) @@ -15,9 +14,9 @@ - - - + + + diff --git a/src/Runner.Worker/Runner.Worker.csproj b/src/Runner.Worker/Runner.Worker.csproj index 53c1610df3e..eee59b8721c 100644 --- a/src/Runner.Worker/Runner.Worker.csproj +++ b/src/Runner.Worker/Runner.Worker.csproj @@ -1,12 +1,11 @@ - net8.0 + net6.0 Exe win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 - true true - NU1701;NU1603;SYSLIB0050;SYSLIB0051 + NU1701;NU1603 $(Version) false true @@ -19,9 +18,9 @@ - - - + + + diff --git a/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs b/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs index 72367654bf1..34c97b73a1d 100644 --- a/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs +++ b/src/Sdk/Common/Common/Exceptions/PropertyExceptions.cs @@ -34,7 +34,6 @@ protected VssPropertyValidationException(SerializationInfo info, StreamingContex public String PropertyName { get; set; } - [Obsolete] [SecurityCritical] public override void GetObjectData(SerializationInfo info, StreamingContext context) { diff --git a/src/Sdk/Common/Common/VssException.cs b/src/Sdk/Common/Common/VssException.cs index 5f8fb8c2554..7cead78655d 100644 --- a/src/Sdk/Common/Common/VssException.cs +++ b/src/Sdk/Common/Common/VssException.cs @@ -127,7 +127,6 @@ protected VssException(SerializationInfo info, StreamingContext context) EventId = (int)info.GetValue("m_eventId", typeof(int)); } - [Obsolete] [SecurityCritical] public override void GetObjectData(SerializationInfo info, StreamingContext context) { diff --git a/src/Sdk/Sdk.csproj b/src/Sdk/Sdk.csproj index 7ba739498a4..ff1cb85a4fe 100644 --- a/src/Sdk/Sdk.csproj +++ b/src/Sdk/Sdk.csproj @@ -1,12 +1,11 @@ - net8.0 + net6.0 Library win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 - true - NU1701;NU1603;SYSLIB0050;SYSLIB0051 + NU1701;NU1603 $(Version) TRACE 8.0 @@ -15,13 +14,13 @@ - + - - - - - + + + + + diff --git a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs index 34ec103d05f..5ebf86f9a8d 100644 --- a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs +++ b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthExceptions.cs @@ -85,7 +85,6 @@ public String Error set; } - [Obsolete] public override void GetObjectData(SerializationInfo info, StreamingContext context) { base.GetObjectData(info, context); diff --git a/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs b/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs index 8dc275c0098..e4aa84a736d 100644 --- a/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs +++ b/src/Sdk/WebApi/WebApi/VssServiceResponseException.cs @@ -24,7 +24,6 @@ protected VssServiceResponseException(SerializationInfo info, StreamingContext c HttpStatusCode = (HttpStatusCode)info.GetInt32("HttpStatusCode"); } - [Obsolete] [SecurityCritical] public override void GetObjectData(SerializationInfo info, StreamingContext context) { diff --git a/src/Test/Test.csproj b/src/Test/Test.csproj index aebe242096f..1beddbfc251 100644 --- a/src/Test/Test.csproj +++ b/src/Test/Test.csproj @@ -1,9 +1,9 @@ - net8.0 + net6.0 win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true - NU1701;NU1603;NU1603;xUnit2013;SYSLIB0050;SYSLIB0051 + NU1701;NU1603;NU1603;xUnit2013; @@ -15,13 +15,13 @@ - + - + - + - + diff --git a/src/dev.sh b/src/dev.sh index 8120ef33267..0fc58761e53 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -17,7 +17,7 @@ LAYOUT_DIR="$SCRIPT_DIR/../_layout" DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x" PACKAGE_DIR="$SCRIPT_DIR/../_package" DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk" -DOTNETSDK_VERSION="8.0.303" +DOTNETSDK_VERSION="6.0.421" DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION" RUNNER_VERSION=$(cat runnerversion) diff --git a/src/global.json b/src/global.json index fd07d882ad3..e7028fe0dd4 100644 --- a/src/global.json +++ b/src/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "8.0.303" + "version": "6.0.421" } } From 7303cb56735ea8969f2785fa1b19d7ce9510eca7 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 6 Aug 2024 18:20:54 -0400 Subject: [PATCH 48/68] Ignore ssl cert on websocket client. (#3423) --- src/Runner.Common/JobServer.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index ec90d879cce..eac20a2b983 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net.Http; using System.Net.Http.Headers; +using System.Net.Security; using System.Net.WebSockets; using System.Text; using System.Threading; @@ -179,6 +180,10 @@ private void InitializeWebsocketClient(TimeSpan delay) userAgentValues.AddRange(UserAgentUtility.GetDefaultRestUserAgent()); userAgentValues.AddRange(HostContext.UserAgents); this._websocketClient.Options.SetRequestHeader("User-Agent", string.Join(" ", userAgentValues.Select(x => x.ToString()))); + if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY"))) + { + this._websocketClient.Options.RemoteCertificateValidationCallback = (_, _, _, _) => true; + } this._websocketConnectTask = ConnectWebSocketClient(feedStreamUrl, delay); } From fb6d1adb430a6c96407c5857854a40662af439c8 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Wed, 7 Aug 2024 16:53:00 -0500 Subject: [PATCH 49/68] .NET 8 OS compatibility test (#3422) * Revert "Warn for soon to be deprecated OS versions (#3413)" This reverts commit ae04147f960f2232d0a888c89d3e128f92f21677. * Add .NET 8 OS compatibility test * feedback --- src/Runner.Common/Constants.cs | 2 + src/Runner.Worker/JobExtension.cs | 2 +- src/Runner.Worker/OSWarningChecker.cs | 113 ++++---- src/Runner.Worker/Variables.cs | 4 + .../Pipelines/AgentJobRequestMessage.cs | 26 -- src/Sdk/DTPipelines/Pipelines/OSWarning.cs | 44 --- src/Test/L0/Listener/JobDispatcherL0.cs | 3 +- src/Test/L0/Listener/RunnerL0.cs | 2 +- src/Test/L0/Worker/ActionCommandManagerL0.cs | 2 +- .../L0/Worker/CreateStepSummaryCommandL0.cs | 2 +- src/Test/L0/Worker/ExecutionContextL0.cs | 32 +-- src/Test/L0/Worker/JobExtensionL0.cs | 2 +- src/Test/L0/Worker/JobRunnerL0.cs | 1 - src/Test/L0/Worker/OSWarningCheckerL0.cs | 271 ------------------ src/Test/L0/Worker/WorkerL0.cs | 2 +- src/TestDotNet8Compatibility/Program.cs | 13 + .../TestDotNet8Compatibility.csproj | 19 ++ src/TestDotNet8Compatibility/dir.proj | 22 ++ src/TestDotNet8Compatibility/global.json | 5 + src/dev.sh | 54 ++++ 20 files changed, 203 insertions(+), 418 deletions(-) delete mode 100644 src/Sdk/DTPipelines/Pipelines/OSWarning.cs delete mode 100644 src/Test/L0/Worker/OSWarningCheckerL0.cs create mode 100644 src/TestDotNet8Compatibility/Program.cs create mode 100644 src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj create mode 100644 src/TestDotNet8Compatibility/dir.proj create mode 100644 src/TestDotNet8Compatibility/global.json diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index d68d5cdf695..849228b8741 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -280,6 +280,8 @@ public static class System public static readonly string PhaseDisplayName = "system.phaseDisplayName"; public static readonly string JobRequestType = "system.jobRequestType"; public static readonly string OrchestrationId = "system.orchestrationId"; + public static readonly string TestDotNet8Compatibility = "system.testDotNet8Compatibility"; + public static readonly string DotNet8CompatibilityWarning = "system.dotNet8CompatibilityWarning"; } } diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 4ca06b90c94..a36e4beb4d8 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -129,7 +129,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel // Check OS warning var osWarningChecker = HostContext.GetService(); - await osWarningChecker.CheckOSAsync(context, message.OSWarnings); + await osWarningChecker.CheckOSAsync(context); try { diff --git a/src/Runner.Worker/OSWarningChecker.cs b/src/Runner.Worker/OSWarningChecker.cs index 900f8f1f0eb..bd1ccc70297 100644 --- a/src/Runner.Worker/OSWarningChecker.cs +++ b/src/Runner.Worker/OSWarningChecker.cs @@ -1,10 +1,9 @@ using System; using System.Collections.Generic; using System.IO; +using System.Threading; using System.Threading.Tasks; -using System.Text.RegularExpressions; using GitHub.DistributedTask.WebApi; -using GitHub.DistributedTask.Pipelines; using GitHub.Runner.Common; using GitHub.Runner.Sdk; @@ -13,75 +12,85 @@ namespace GitHub.Runner.Worker [ServiceLocator(Default = typeof(OSWarningChecker))] public interface IOSWarningChecker : IRunnerService { - Task CheckOSAsync(IExecutionContext context, IList osWarnings); + Task CheckOSAsync(IExecutionContext context); } -#if OS_WINDOWS || OS_OSX public sealed class OSWarningChecker : RunnerService, IOSWarningChecker { - public Task CheckOSAsync(IExecutionContext context, IList osWarnings) + public async Task CheckOSAsync(IExecutionContext context) { ArgUtil.NotNull(context, nameof(context)); - ArgUtil.NotNull(osWarnings, nameof(osWarnings)); - return Task.CompletedTask; - } - } -#else - public sealed class OSWarningChecker : RunnerService, IOSWarningChecker - { - private static readonly TimeSpan s_matchTimeout = TimeSpan.FromMilliseconds(100); - private static readonly RegexOptions s_regexOptions = RegexOptions.CultureInvariant | RegexOptions.IgnoreCase; - - public async Task CheckOSAsync(IExecutionContext context, IList osWarnings) - { - ArgUtil.NotNull(context, nameof(context)); - ArgUtil.NotNull(osWarnings, nameof(osWarnings)); - foreach (var osWarning in osWarnings) + if (!context.Global.Variables.System_TestDotNet8Compatibility) { - if (string.IsNullOrEmpty(osWarning.FilePath)) - { - Trace.Error("The file path is not specified in the OS warning check."); - continue; - } + return; + } - if (string.IsNullOrEmpty(osWarning.RegularExpression)) + context.Output("Testing runner upgrade compatibility"); + List output = new(); + object outputLock = new(); + try + { + using (var process = HostContext.CreateService()) { - Trace.Error("The regular expression is not specified in the OS warning check."); - continue; - } + process.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout) + { + if (!string.IsNullOrEmpty(stdout.Data)) + { + lock (outputLock) + { + output.Add(stdout.Data); + Trace.Info(stdout.Data); + } + } + }; - if (string.IsNullOrEmpty(osWarning.Warning)) - { - Trace.Error("The warning message is not specified in the OS warning check."); - continue; - } + process.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr) + { + if (!string.IsNullOrEmpty(stderr.Data)) + { + lock (outputLock) + { + output.Add(stderr.Data); + Trace.Error(stderr.Data); + } + } + }; - try - { - if (File.Exists(osWarning.FilePath)) + using (var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10))) { - var lines = await File.ReadAllLinesAsync(osWarning.FilePath, context.CancellationToken); - var regex = new Regex(osWarning.RegularExpression, s_regexOptions, s_matchTimeout); - foreach (var line in lines) + int exitCode = await process.ExecuteAsync( + workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root), + fileName: Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "testDotNet8Compatibility", $"TestDotNet8Compatibility{IOUtil.ExeExtension}"), + arguments: string.Empty, + environment: null, + cancellationToken: cancellationTokenSource.Token); + + var outputStr = string.Join("\n", output).Trim(); + if (exitCode != 0 || !string.Equals(outputStr, "Hello from .NET 8!", StringComparison.Ordinal)) { - if (regex.IsMatch(line)) + var warningMessage = context.Global.Variables.System_DotNet8CompatibilityWarning; + if (!string.IsNullOrEmpty(warningMessage)) { - context.Warning(osWarning.Warning); - context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $"OS warning: {osWarning.Warning}" }); - return; + context.Warning(warningMessage); } + + context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test failed with exit code '{exitCode}' and output: {GetShortOutput(output)}" }); } } } - catch (Exception ex) - { - Trace.Error("An error occurred while checking OS warnings for file '{0}' and regex '{1}'.", osWarning.FilePath, osWarning.RegularExpression); - Trace.Error(ex); - context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $"An error occurred while checking OS warnings for file '{osWarning.FilePath}' and regex '{osWarning.RegularExpression}': {ex.Message}" }); - } + } + catch (Exception ex) + { + Trace.Error("An error occurred while testing .NET 8 compatibility'"); + Trace.Error(ex); + context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test encountered exception type '{ex.GetType().FullName}', message: '{ex.Message}', process output: '{GetShortOutput(output)}'" }); } } + + private static string GetShortOutput(List output) + { + var outputStr = string.Join("\n", output).Trim(); + return outputStr.Length > 200 ? string.Concat(outputStr.Substring(0, 200), "[...]") : outputStr; + } } -#endif } - diff --git a/src/Runner.Worker/Variables.cs b/src/Runner.Worker/Variables.cs index 916b82dc6a1..a30cd3c39e7 100644 --- a/src/Runner.Worker/Variables.cs +++ b/src/Runner.Worker/Variables.cs @@ -72,8 +72,12 @@ public Variables(IHostContext hostContext, IDictionary co public bool? Step_Debug => GetBoolean(Constants.Variables.Actions.StepDebug); + public string System_DotNet8CompatibilityWarning => Get(Constants.Variables.System.DotNet8CompatibilityWarning); + public string System_PhaseDisplayName => Get(Constants.Variables.System.PhaseDisplayName); + public bool System_TestDotNet8Compatibility => GetBoolean(Constants.Variables.System.TestDotNet8Compatibility) ?? false; + public string Get(string name) { Variable variable; diff --git a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs index 030f1d82610..070d86ee24f 100644 --- a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs +++ b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs @@ -44,7 +44,6 @@ public AgentJobRequestMessage( IList defaults, ActionsEnvironmentReference actionsEnvironment, TemplateToken snapshot, - IList osWarnings, String messageType = JobRequestMessageTypes.PipelineAgentJobRequest) { this.MessageType = messageType; @@ -74,11 +73,6 @@ public AgentJobRequestMessage( m_defaults = new List(defaults); } - if (osWarnings?.Count > 0) - { - m_osWarnings = new List(osWarnings); - } - this.ContextData = new Dictionary(StringComparer.OrdinalIgnoreCase); if (contextData?.Count > 0) { @@ -294,18 +288,6 @@ public IList FileTable } } - public IList OSWarnings - { - get - { - if (m_osWarnings == null) - { - m_osWarnings = new List(); - } - return m_osWarnings; - } - } - // todo: remove after feature-flag DistributedTask.EvaluateContainerOnRunner is enabled everywhere public void SetJobSidecarContainers(IDictionary value) { @@ -443,11 +425,6 @@ private void OnSerializing(StreamingContext context) { JobContainer = new StringToken(null, null, null, m_jobContainerResourceAlias); } - - if (m_osWarnings?.Count == 0) - { - m_osWarnings = null; - } } [DataMember(Name = "EnvironmentVariables", EmitDefaultValue = false)] @@ -472,9 +449,6 @@ private void OnSerializing(StreamingContext context) [DataMember(Name = "JobSidecarContainers", EmitDefaultValue = false)] private IDictionary m_jobSidecarContainers; - [DataMember(Name = "OSWarnings", EmitDefaultValue = false)] - private List m_osWarnings; - // todo: remove after feature-flag DistributedTask.EvaluateContainerOnRunner is enabled everywhere [IgnoreDataMember] private string m_jobContainerResourceAlias; diff --git a/src/Sdk/DTPipelines/Pipelines/OSWarning.cs b/src/Sdk/DTPipelines/Pipelines/OSWarning.cs deleted file mode 100644 index 7eb4168b26f..00000000000 --- a/src/Sdk/DTPipelines/Pipelines/OSWarning.cs +++ /dev/null @@ -1,44 +0,0 @@ -using System; -using System.ComponentModel; -using System.Runtime.Serialization; - -namespace GitHub.DistributedTask.Pipelines -{ - /// - /// Information to check whether the OS is going to be deprecated soon - /// - [DataContract] - [EditorBrowsable(EditorBrowsableState.Never)] - public sealed class OSWarning - { - /// - /// Gets or sets the file to check - /// - [DataMember(EmitDefaultValue = false)] - public String FilePath - { - get; - set; - } - - /// - /// Gets or sets the regular expression to match - /// - [DataMember(EmitDefaultValue = false)] - public String RegularExpression - { - get; - set; - } - - /// - /// Gets or sets the warning annotation message, if the regular expression matches the content of the file - /// - [DataMember(EmitDefaultValue = false)] - public String Warning - { - get; - set; - } - } -} diff --git a/src/Test/L0/Listener/JobDispatcherL0.cs b/src/Test/L0/Listener/JobDispatcherL0.cs index c1d5346fe0a..cc50c180456 100644 --- a/src/Test/L0/Listener/JobDispatcherL0.cs +++ b/src/Test/L0/Listener/JobDispatcherL0.cs @@ -41,7 +41,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage() TaskOrchestrationPlanReference plan = new(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); - var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); result.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); return result; } @@ -810,7 +810,6 @@ private static AgentJobRequestMessage GetAgentJobRequestMessage() null, new List(), new ActionsEnvironmentReference("env"), - null, null ); return message; diff --git a/src/Test/L0/Listener/RunnerL0.cs b/src/Test/L0/Listener/RunnerL0.cs index 3010dcb402a..b29f8835c2b 100644 --- a/src/Test/L0/Listener/RunnerL0.cs +++ b/src/Test/L0/Listener/RunnerL0.cs @@ -44,7 +44,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName) TaskOrchestrationPlanReference plan = new(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); - return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); } private JobCancelMessage CreateJobCancelMessage() diff --git a/src/Test/L0/Worker/ActionCommandManagerL0.cs b/src/Test/L0/Worker/ActionCommandManagerL0.cs index 8be95ae4da5..3a1f8f70fe9 100644 --- a/src/Test/L0/Worker/ActionCommandManagerL0.cs +++ b/src/Test/L0/Worker/ActionCommandManagerL0.cs @@ -232,7 +232,7 @@ public void EchoProcessCommandDebugOn() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs b/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs index 972d6843bd7..185f44b3865 100644 --- a/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs +++ b/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs @@ -193,7 +193,7 @@ private TestHostContext Setup([CallerMemberName] string name = "") TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "Summary Job"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 0054c969938..08abcd09585 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -29,7 +29,7 @@ public void AddIssue_CountWarningsErrors() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -106,7 +106,7 @@ public void ApplyContinueOnError_CheckResultAndOutcome() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -162,7 +162,7 @@ public void AddIssue_TrimMessageSize() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -216,7 +216,7 @@ public void AddIssue_OverrideLogMessage() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -271,7 +271,7 @@ public void AddIssue_AddStepAndLineNumberInformation() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -322,7 +322,7 @@ public void Debug_Multilines() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -373,7 +373,7 @@ public void RegisterPostJobAction_ShareState() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -471,7 +471,7 @@ public void RegisterPostJobAction_NotRegisterPostTwice() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -555,7 +555,7 @@ public void ActionResult_Lowercase() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -610,7 +610,7 @@ public void PublishStepTelemetry_RegularStep_NoOpt() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -653,7 +653,7 @@ public void PublishStepTelemetry_RegularStep() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -717,7 +717,7 @@ public void PublishStepTelemetry_EmbeddedStep() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -781,7 +781,7 @@ public void PublishStepResult_EmbeddedStep() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -969,7 +969,7 @@ public void ActionVariables_AddedToVarsContext() TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -1014,7 +1014,7 @@ public void ActionVariables_DebugUsingVars() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -1057,7 +1057,7 @@ public void ActionVariables_SecretsPrecedenceForDebugUsingVars() TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index 6ada9143dc3..4f5834fbe69 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -114,7 +114,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " }; Guid jobId = Guid.NewGuid(); - _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null, null, null); + _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null, null); GitHubContext github = new(); github["repository"] = new Pipelines.ContextData.StringContextData("actions/runner"); github["secret_source"] = new Pipelines.ContextData.StringContextData("Actions"); diff --git a/src/Test/L0/Worker/JobRunnerL0.cs b/src/Test/L0/Worker/JobRunnerL0.cs index c1021150c6f..e8011b9b051 100644 --- a/src/Test/L0/Worker/JobRunnerL0.cs +++ b/src/Test/L0/Worker/JobRunnerL0.cs @@ -102,7 +102,6 @@ private Pipelines.AgentJobRequestMessage GetMessage(String messageType = JobRequ testName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, new ActionsEnvironmentReference("staging"), null, - null, messageType: messageType); message.Variables[Constants.Variables.System.Culture] = "en-US"; message.Resources.Endpoints.Add(new ServiceEndpoint() diff --git a/src/Test/L0/Worker/OSWarningCheckerL0.cs b/src/Test/L0/Worker/OSWarningCheckerL0.cs deleted file mode 100644 index 7f4d0f32ee2..00000000000 --- a/src/Test/L0/Worker/OSWarningCheckerL0.cs +++ /dev/null @@ -1,271 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using GitHub.DistributedTask.Pipelines; -using GitHub.DistributedTask.WebApi; -using GitHub.Runner.Worker; -using Moq; -using Xunit; - -namespace GitHub.Runner.Common.Tests.Worker -{ - public sealed class OSWarningCheckerL0 - { - private CancellationTokenSource _ecTokenSource; - private Mock _ec; - private TestHostContext _hc; - private OSWarningChecker _osWarningChecker; - private List _issues; - private string _workFolder; - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void CheckOS_FileNotExists() - { - try - { - // Arrange - Setup(); - var osWarnings = new List - { - new OSWarning - { - FilePath = Path.Combine(_workFolder, "os-release"), - RegularExpression = "some OS version", - Warning = "Some OS version will be deprecated soon" - }, - }; - - // Act - await _osWarningChecker.CheckOSAsync(_ec.Object, osWarnings); - - // Assert - Assert.Equal(0, _issues.Count); - Assert.Equal(0, _ec.Object.Global.JobTelemetry.Count); - } - finally - { - Teardown(); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void CheckOS_CaseInsensitive() - { - try - { - // Arrange - Setup(); - var osWarnings = new List - { - new OSWarning - { - FilePath = Path.Combine(_workFolder, "os-release"), - RegularExpression = "some OS verSION", - Warning = "Some OS version will be deprecated soon" - }, - }; - File.WriteAllText(Path.Combine(_workFolder, "os-release"), "some OS version\n"); - - // Act - await _osWarningChecker.CheckOSAsync(_ec.Object, osWarnings); - -#if OS_WINDOWS || OS_OSX - // Assert - Assert.Equal(0, _issues.Count); - Assert.Equal(0, _ec.Object.Global.JobTelemetry.Count); -#else - // Assert - Assert.Equal(1, _issues.Count); - Assert.Equal(IssueType.Warning, _issues[0].Type); - Assert.Equal("Some OS version will be deprecated soon", _issues[0].Message); - Assert.Equal(1, _ec.Object.Global.JobTelemetry.Count); - Assert.Equal(JobTelemetryType.General, _ec.Object.Global.JobTelemetry[0].Type); - Assert.Equal("OS warning: Some OS version will be deprecated soon", _ec.Object.Global.JobTelemetry[0].Message); -#endif - } - finally - { - Teardown(); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void CheckOS_MatchesOnceWithinAFile() - { - try - { - // Arrange - Setup(); - var osWarnings = new List - { - new OSWarning - { - FilePath = Path.Combine(_workFolder, "os-release"), - RegularExpression = "some OS version", - Warning = "Some OS version will be deprecated soon" - }, - }; - File.WriteAllText(Path.Combine(_workFolder, "os-release"), "some OS version\nsome OS version\n"); - - // Act - await _osWarningChecker.CheckOSAsync(_ec.Object, osWarnings); - -#if OS_WINDOWS || OS_OSX - // Assert - Assert.Equal(0, _issues.Count); - Assert.Equal(0, _ec.Object.Global.JobTelemetry.Count); -#else - // Assert - Assert.Equal(1, _issues.Count); - Assert.Equal(IssueType.Warning, _issues[0].Type); - Assert.Equal("Some OS version will be deprecated soon", _issues[0].Message); - Assert.Equal(1, _ec.Object.Global.JobTelemetry.Count); - Assert.Equal(JobTelemetryType.General, _ec.Object.Global.JobTelemetry[0].Type); - Assert.Equal("OS warning: Some OS version will be deprecated soon", _ec.Object.Global.JobTelemetry[0].Message); -#endif - } - finally - { - Teardown(); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void CheckOS_MatchesOnceAcrossFiles() - { - try - { - // Arrange - Setup(); - var osWarnings = new List - { - new OSWarning - { - FilePath = Path.Combine(_workFolder, "os-release"), - RegularExpression = "some OS version", - Warning = "Some OS version will be deprecated soon" - }, - new OSWarning - { - FilePath = Path.Combine(_workFolder, "os-release-2"), - RegularExpression = "some OS version", - Warning = "Some OS version will be deprecated soon" - }, - }; - File.WriteAllText(Path.Combine(_workFolder, "os-release"), "some OS version\n"); - File.WriteAllText(Path.Combine(_workFolder, "os-release-2"), "some OS version\n"); - - // Act - await _osWarningChecker.CheckOSAsync(_ec.Object, osWarnings); - -#if OS_WINDOWS || OS_OSX - // Assert - Assert.Equal(0, _issues.Count); - Assert.Equal(0, _ec.Object.Global.JobTelemetry.Count); -#else - // Assert - Assert.Equal(1, _issues.Count); - Assert.Equal(IssueType.Warning, _issues[0].Type); - Assert.Equal("Some OS version will be deprecated soon", _issues[0].Message); - Assert.Equal(1, _ec.Object.Global.JobTelemetry.Count); - Assert.Equal(JobTelemetryType.General, _ec.Object.Global.JobTelemetry[0].Type); - Assert.Equal("OS warning: Some OS version will be deprecated soon", _ec.Object.Global.JobTelemetry[0].Message); -#endif - } - finally - { - Teardown(); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void CheckOS_LogsTelemetryUponException() - { - try - { - // Arrange - Setup(); - var osWarnings = new List - { - new OSWarning - { - FilePath = Path.Combine(_workFolder, "os-release"), - RegularExpression = "abc[", // Invalid pattern - Warning = "Some OS version will be deprecated soon" - }, - }; - File.WriteAllText(Path.Combine(_workFolder, "os-release"), "some OS version\n"); - - // Act - await _osWarningChecker.CheckOSAsync(_ec.Object, osWarnings); - -#if OS_WINDOWS || OS_OSX - // Assert - Assert.Equal(0, _issues.Count); - Assert.Equal(0, _ec.Object.Global.JobTelemetry.Count); -#else - // Assert - Assert.Equal(0, _issues.Count); - Assert.Equal(1, _ec.Object.Global.JobTelemetry.Count); - Assert.Equal(JobTelemetryType.General, _ec.Object.Global.JobTelemetry[0].Type); - Assert.Equal( - $"An error occurred while checking OS warnings for file '{osWarnings[0].FilePath}' and regex '{osWarnings[0].RegularExpression}': Invalid pattern 'abc[' at offset 4. Unterminated [] set.", - _ec.Object.Global.JobTelemetry[0].Message); -#endif - } - finally - { - Teardown(); - } - } - - private void Setup([CallerMemberName] string name = "") - { - _issues = new List(); - - // Test host context - _hc = new TestHostContext(this, name); - - // Random work folder - _workFolder = _hc.GetDirectory(WellKnownDirectory.Work); - Directory.CreateDirectory(_workFolder); - - // Execution context token source - _ecTokenSource?.Dispose(); - _ecTokenSource = new CancellationTokenSource(); - - // Execution context - _ec = new Mock(); - _ec.Setup(x => x.Global).Returns(new GlobalContext()); - _ec.Setup(x => x.CancellationToken).Returns(_ecTokenSource.Token); - _ec.Object.Global.JobTelemetry = new List(); - _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, ExecutionContextLogOptions logOptions) => { _issues.Add(issue); }); - - // OS warning checker - _osWarningChecker = new OSWarningChecker(); - _osWarningChecker.Initialize(_hc); - } - - private void Teardown() - { - _hc?.Dispose(); - if (!string.IsNullOrEmpty(_workFolder) && Directory.Exists(_workFolder)) - { - Directory.Delete(_workFolder, recursive: true); - } - } - } -} diff --git a/src/Test/L0/Worker/WorkerL0.cs b/src/Test/L0/Worker/WorkerL0.cs index ba6a933afdd..defcc981404 100644 --- a/src/Test/L0/Worker/WorkerL0.cs +++ b/src/Test/L0/Worker/WorkerL0.cs @@ -67,7 +67,7 @@ private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName) new Pipelines.ContextData.DictionaryContextData() }, }; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List(), resources, context, null, actions, null, null, null, null, null, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List(), resources, context, null, actions, null, null, null, null, null); return jobRequest; } diff --git a/src/TestDotNet8Compatibility/Program.cs b/src/TestDotNet8Compatibility/Program.cs new file mode 100644 index 00000000000..0d231953003 --- /dev/null +++ b/src/TestDotNet8Compatibility/Program.cs @@ -0,0 +1,13 @@ +using System; + +namespace TestDotNet8Compatibility +{ + public static class Program + { + public static int Main(string[] args) + { + Console.WriteLine("Hello from .NET 8!"); + return 0; + } + } +} diff --git a/src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj b/src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj new file mode 100644 index 00000000000..5fb8e44a1cf --- /dev/null +++ b/src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj @@ -0,0 +1,19 @@ + + + + net8.0 + Exe + win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 + true + true + true + $(Version) + false + true + + + + portable + + + diff --git a/src/TestDotNet8Compatibility/dir.proj b/src/TestDotNet8Compatibility/dir.proj new file mode 100644 index 00000000000..fa8200ba95a --- /dev/null +++ b/src/TestDotNet8Compatibility/dir.proj @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/src/TestDotNet8Compatibility/global.json b/src/TestDotNet8Compatibility/global.json new file mode 100644 index 00000000000..fd07d882ad3 --- /dev/null +++ b/src/TestDotNet8Compatibility/global.json @@ -0,0 +1,5 @@ +{ + "sdk": { + "version": "8.0.303" + } +} diff --git a/src/dev.sh b/src/dev.sh index 0fc58761e53..71c80637c02 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -19,6 +19,8 @@ PACKAGE_DIR="$SCRIPT_DIR/../_package" DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk" DOTNETSDK_VERSION="6.0.421" DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION" +DOTNET8SDK_VERSION="8.0.303" +DOTNET8SDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNET8SDK_VERSION" RUNNER_VERSION=$(cat runnerversion) pushd "$SCRIPT_DIR" @@ -125,6 +127,19 @@ function build () { heading "Building ..." dotnet msbuild -t:Build -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed build + + # Build TestDotNet8Compatibility + heading "Building .NET 8 compatibility test" + echo "Prepend ${DOTNET8SDK_INSTALLDIR} to %PATH%" # Prepend .NET 8 SDK to PATH + PATH_BAK=$PATH + export PATH=${DOTNET8SDK_INSTALLDIR}:$PATH + pushd "$SCRIPT_DIR/TestDotNet8Compatibility" > /dev/null # Working directory + pwd + echo "Dotnet 8 SDK Version" + dotnet --version + dotnet msbuild -t:Build -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed build + popd > /dev/null # Restore working directory + export PATH=$PATH_BAK # Restore PATH } function layout () @@ -143,6 +158,18 @@ function layout () heading "Setup externals folder for $RUNTIME_ID runner's layout" bash ./Misc/externals.sh $RUNTIME_ID || checkRC externals.sh + + # Build TestDotNet8Compatibility + echo "Prepend ${DOTNET8SDK_INSTALLDIR} to %PATH%" # Prepend .NET 8 SDK to PATH + PATH_BAK=$PATH + export PATH=${DOTNET8SDK_INSTALLDIR}:$PATH + pushd "$SCRIPT_DIR/TestDotNet8Compatibility" > /dev/null # Working directory + heading "Dotnet 8 SDK Version" + dotnet --version + heading "Building .NET 8 compatibility test" + dotnet msbuild -t:layout -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed build + popd > /dev/null # Restore working directory + export PATH=$PATH_BAK # Restore PATH } function runtest () @@ -199,6 +226,7 @@ function package () popd > /dev/null } +# Install .NET SDK if [[ (! -d "${DOTNETSDK_INSTALLDIR}") || (! -e "${DOTNETSDK_INSTALLDIR}/.${DOTNETSDK_VERSION}") || (! -e "${DOTNETSDK_INSTALLDIR}/dotnet") ]]; then # Download dotnet SDK to ../_dotnetsdk directory @@ -224,6 +252,32 @@ if [[ (! -d "${DOTNETSDK_INSTALLDIR}") || (! -e "${DOTNETSDK_INSTALLDIR}/.${DOTN echo "${DOTNETSDK_VERSION}" > "${DOTNETSDK_INSTALLDIR}/.${DOTNETSDK_VERSION}" fi +# Install .NET 8 SDK +if [[ (! -d "${DOTNET8SDK_INSTALLDIR}") || (! -e "${DOTNET8SDK_INSTALLDIR}/.${DOTNET8SDK_VERSION}") || (! -e "${DOTNET8SDK_INSTALLDIR}/dotnet") ]]; then + + # Download dotnet 8 SDK to ../_dotnetsdk directory + heading "Ensure Dotnet 8 SDK" + + # _dotnetsdk + # \1.0.x + # \dotnet + # \.1.0.x + echo "Download dotnet8sdk into ${DOTNET8SDK_INSTALLDIR}" + rm -Rf "${DOTNETSDK_DIR}" + + # run dotnet-install.ps1 on windows, dotnet-install.sh on linux + if [[ ("$CURRENT_PLATFORM" == "windows") ]]; then + echo "Convert ${DOTNET8SDK_INSTALLDIR} to Windows style path" + sdkinstallwindow_path=${DOTNET8SDK_INSTALLDIR:1} + sdkinstallwindow_path=${sdkinstallwindow_path:0:1}:${sdkinstallwindow_path:1} + $POWERSHELL -NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command "& \"./Misc/dotnet-install.ps1\" -Version ${DOTNET8SDK_VERSION} -InstallDir \"${sdkinstallwindow_path}\" -NoPath; exit \$LastExitCode;" || checkRC dotnet-install.ps1 + else + bash ./Misc/dotnet-install.sh --version ${DOTNET8SDK_VERSION} --install-dir "${DOTNET8SDK_INSTALLDIR}" --no-path || checkRC dotnet-install.sh + fi + + echo "${DOTNET8SDK_VERSION}" > "${DOTNET8SDK_INSTALLDIR}/.${DOTNET8SDK_VERSION}" +fi + echo "Prepend ${DOTNETSDK_INSTALLDIR} to %PATH%" export PATH=${DOTNETSDK_INSTALLDIR}:$PATH From 7e84ae0b30ceb0c0a00a9194b5cdca2c004711c0 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Thu, 8 Aug 2024 08:57:44 -0500 Subject: [PATCH 50/68] Prepare release 2.319.0 (#3424) --- releaseNote.md | 18 +++++------------- src/runnerversion | 2 +- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 5a2f71142f6..9753b6d2311 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,18 +1,10 @@ ## What's Changed -- Update Docker to v27.1.1 by @TingluoHuang in https://github.com/actions/runner/pull/3401 -- Upgrade dotnet sdk to v8.0.303 in https://github.com/actions/runner/pull/3388 -- Rephrase node20 warning by @rentziass in https://github.com/actions/runner/pull/3376 -- Bump hook version to 0.6.1 by @nikola-jokic in https://github.com/actions/runner/pull/3350 -- Backoff to avoid excessive retries to Run Service in a duration by @ericsciple in https://github.com/actions/runner/pull/3354 -- Bump System.Security.Cryptography.Pkcs from 5.0.0 to 8.0.0 in /src in https://github.com/actions/runner/pull/3347 -- Upgrade dotnet sdk to v8.0.302 in https://github.com/actions/runner/pull/3346 -- Bump runner to dotnet 8 by @TingluoHuang in https://github.com/actions/runner/pull/3345 -- Pass runner version as environment variable in workflow by @joshmgross in https://github.com/actions/runner/pull/3318 -- Make sure we mask secrets when reporting telemetry by @TingluoHuang in https://github.com/actions/runner/pull/3315 -- Bump docker version and docker buildx version by @int128 in https://github.com/actions/runner/pull/3277 - -**Full Changelog**: https://github.com/actions/runner/compare/v2.317.0...v2.318.0 +- .NET 8 OS compatibility test https://github.com/actions/runner/pull/3422 +- Ignore ssl cert on websocket client https://github.com/actions/runner/pull/3423 +- Revert "Bump runner to dotnet 8" https://github.com/actions/runner/pull/3412 + +**Full Changelog**: https://github.com/actions/runner/compare/v2.318.0...v2.319.0 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. diff --git a/src/runnerversion b/src/runnerversion index 4d0a47ffd8e..8330b5bf1db 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.318.0 +2.319.0 From a77fe8a53fe9fc2a77ae4f511d46d04197bdde66 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Tue, 13 Aug 2024 09:02:26 -0500 Subject: [PATCH 51/68] .NET 8 compat test adjustments: 1) do not trim SDK, 2) support pattern to match output, 3) modify output truncation length (#3427) --- src/Runner.Common/Constants.cs | 2 ++ src/Runner.Worker/OSWarningChecker.cs | 22 +++++++++++++++---- src/Runner.Worker/Variables.cs | 4 ++++ .../TestDotNet8Compatibility.csproj | 1 - 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 849228b8741..383ec7a10f5 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -281,6 +281,8 @@ public static class System public static readonly string JobRequestType = "system.jobRequestType"; public static readonly string OrchestrationId = "system.orchestrationId"; public static readonly string TestDotNet8Compatibility = "system.testDotNet8Compatibility"; + public static readonly string DotNet8CompatibilityOutputLength = "system.dotNet8CompatibilityOutputLength"; + public static readonly string DotNet8CompatibilityOutputPattern = "system.dotNet8CompatibilityOutputPattern"; public static readonly string DotNet8CompatibilityWarning = "system.dotNet8CompatibilityWarning"; } } diff --git a/src/Runner.Worker/OSWarningChecker.cs b/src/Runner.Worker/OSWarningChecker.cs index bd1ccc70297..765a41a5215 100644 --- a/src/Runner.Worker/OSWarningChecker.cs +++ b/src/Runner.Worker/OSWarningChecker.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using GitHub.DistributedTask.WebApi; @@ -17,6 +18,8 @@ public interface IOSWarningChecker : IRunnerService public sealed class OSWarningChecker : RunnerService, IOSWarningChecker { + private static TimeSpan s_regexTimeout = TimeSpan.FromSeconds(1); + public async Task CheckOSAsync(IExecutionContext context) { ArgUtil.NotNull(context, nameof(context)); @@ -68,13 +71,23 @@ public async Task CheckOSAsync(IExecutionContext context) var outputStr = string.Join("\n", output).Trim(); if (exitCode != 0 || !string.Equals(outputStr, "Hello from .NET 8!", StringComparison.Ordinal)) { + var pattern = context.Global.Variables.System_DotNet8CompatibilityOutputPattern; + if (!string.IsNullOrEmpty(pattern)) + { + var regex = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, s_regexTimeout); + if (!regex.IsMatch(outputStr)) + { + return; + } + } + var warningMessage = context.Global.Variables.System_DotNet8CompatibilityWarning; if (!string.IsNullOrEmpty(warningMessage)) { context.Warning(warningMessage); } - context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test failed with exit code '{exitCode}' and output: {GetShortOutput(output)}" }); + context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test failed with exit code '{exitCode}' and output: {GetShortOutput(context, output)}" }); } } } @@ -83,14 +96,15 @@ public async Task CheckOSAsync(IExecutionContext context) { Trace.Error("An error occurred while testing .NET 8 compatibility'"); Trace.Error(ex); - context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test encountered exception type '{ex.GetType().FullName}', message: '{ex.Message}', process output: '{GetShortOutput(output)}'" }); + context.Global.JobTelemetry.Add(new JobTelemetry() { Type = JobTelemetryType.General, Message = $".NET 8 OS compatibility test encountered exception type '{ex.GetType().FullName}', message: '{ex.Message}', process output: '{GetShortOutput(context, output)}'" }); } } - private static string GetShortOutput(List output) + private static string GetShortOutput(IExecutionContext context, List output) { + var length = context.Global.Variables.System_DotNet8CompatibilityOutputLength ?? 200; var outputStr = string.Join("\n", output).Trim(); - return outputStr.Length > 200 ? string.Concat(outputStr.Substring(0, 200), "[...]") : outputStr; + return outputStr.Length > length ? string.Concat(outputStr.Substring(0, length), "[...]") : outputStr; } } } diff --git a/src/Runner.Worker/Variables.cs b/src/Runner.Worker/Variables.cs index a30cd3c39e7..7627ec37984 100644 --- a/src/Runner.Worker/Variables.cs +++ b/src/Runner.Worker/Variables.cs @@ -74,6 +74,10 @@ public Variables(IHostContext hostContext, IDictionary co public string System_DotNet8CompatibilityWarning => Get(Constants.Variables.System.DotNet8CompatibilityWarning); + public string System_DotNet8CompatibilityOutputPattern => Get(Constants.Variables.System.DotNet8CompatibilityOutputPattern); + + public int? System_DotNet8CompatibilityOutputLength => GetInt(Constants.Variables.System.DotNet8CompatibilityOutputLength); + public string System_PhaseDisplayName => Get(Constants.Variables.System.PhaseDisplayName); public bool System_TestDotNet8Compatibility => GetBoolean(Constants.Variables.System.TestDotNet8Compatibility) ?? false; diff --git a/src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj b/src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj index 5fb8e44a1cf..246b690a1ae 100644 --- a/src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj +++ b/src/TestDotNet8Compatibility/TestDotNet8Compatibility.csproj @@ -5,7 +5,6 @@ Exe win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true - true true $(Version) false From 2979fbad9460c32bea9419595d8c3eacc8f4930d Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Fri, 16 Aug 2024 15:17:13 -0400 Subject: [PATCH 52/68] Add pid to user-agent and session owner. (#3432) --- src/Runner.Common/HostContext.cs | 4 ++++ src/Runner.Listener/BrokerMessageListener.cs | 3 ++- src/Runner.Listener/MessageListener.cs | 3 ++- src/runnerversion | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index 78ea8ba4cbe..06b0cd5892a 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -244,6 +244,10 @@ public HostContext(string hostType, string logFile = null) _trace.Info($"Adding extra user agent '{extraUserAgentHeader}' to all HTTP requests."); _userAgents.Add(extraUserAgentHeader); } + + var currentProcess = Process.GetCurrentProcess(); + _userAgents.Add(new ProductInfoHeaderValue("Pid", currentProcess.Id.ToString())); + _userAgents.Add(new ProductInfoHeaderValue("CreationTime", Uri.EscapeDataString(DateTime.UtcNow.ToString("O")))); } public string GetDirectory(WellKnownDirectory directory) diff --git a/src/Runner.Listener/BrokerMessageListener.cs b/src/Runner.Listener/BrokerMessageListener.cs index 8f44fe84372..81ef5402a84 100644 --- a/src/Runner.Listener/BrokerMessageListener.cs +++ b/src/Runner.Listener/BrokerMessageListener.cs @@ -69,7 +69,8 @@ public async Task CreateSessionAsync(CancellationToken toke Version = BuildConstants.RunnerPackage.Version, OSDescription = RuntimeInformation.OSDescription, }; - string sessionName = $"{Environment.MachineName ?? "RUNNER"}"; + var currentProcess = Process.GetCurrentProcess(); + string sessionName = $"{Environment.MachineName ?? "RUNNER"} (PID: {currentProcess.Id})"; var taskAgentSession = new TaskAgentSession(sessionName, agent); string errorMessage = string.Empty; diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 7b51423e6f5..6335667267e 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -88,7 +88,8 @@ public async Task CreateSessionAsync(CancellationToken toke Version = BuildConstants.RunnerPackage.Version, OSDescription = RuntimeInformation.OSDescription, }; - string sessionName = $"{Environment.MachineName ?? "RUNNER"}"; + var currentProcess = Process.GetCurrentProcess(); + string sessionName = $"{Environment.MachineName ?? "RUNNER"} (PID: {currentProcess.Id})"; var taskAgentSession = new TaskAgentSession(sessionName, agent); string errorMessage = string.Empty; diff --git a/src/runnerversion b/src/runnerversion index 8330b5bf1db..4c47f6f2f86 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.319.0 +2.319.1 From e1fa1fcbc3de1c0b57f6fe2a82f77a8ed3a138b1 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Tue, 20 Aug 2024 09:55:30 -0400 Subject: [PATCH 53/68] fix: add `jq`, `git`, `unzip` and `curl` to default packages installed (#3056) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: add `git` and `curl` to default packages installed Hi 👋🏼 These packages are used in a ton of actions on the marketplace. It would be nice if they were installed and ready for use instead of having to install them with `apt-get` on every single Github workflow. * Update Dockerfile * Update images/Dockerfile Co-authored-by: Guillermo Caracuel <633810+gcaracuel@users.noreply.github.com> * Update images/Dockerfile Co-authored-by: Tingluo Huang --------- Co-authored-by: Guillermo Caracuel <633810+gcaracuel@users.noreply.github.com> Co-authored-by: Tingluo Huang --- images/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/images/Dockerfile b/images/Dockerfile index dd8437e1d27..7064053acfb 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -41,12 +41,13 @@ ENV ImageOS=ubuntu22 # 'gpg-agent' and 'software-properties-common' are needed for the 'add-apt-repository' command that follows RUN apt update -y \ - && apt install -y --no-install-recommends sudo lsb-release gpg-agent software-properties-common \ + && apt install -y --no-install-recommends sudo lsb-release gpg-agent software-properties-common curl jq unzip \ && rm -rf /var/lib/apt/lists/* # Configure git-core/ppa based on guidance here: https://git-scm.com/download/linux RUN add-apt-repository ppa:git-core/ppa \ - && apt update -y + && apt update -y \ + && apt install -y --no-install-recommends git RUN adduser --disabled-password --gecos "" --uid 1001 runner \ && groupadd docker --gid 123 \ From 99b464e10200ade9ee2ed20c9b7568c1dabc5d2b Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 27 Aug 2024 12:05:26 -0400 Subject: [PATCH 54/68] Trace GitHub RequestId to log. (#3442) --- src/Runner.Worker/ActionManager.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 0ec72b0585f..f32cad28ea9 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -1102,6 +1102,7 @@ private async Task DownloadRepositoryArchive(IExecutionContext executionContext, int timeoutSeconds = 20 * 60; while (retryCount < 3) { + string requestId = string.Empty; using (var actionDownloadTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds))) using (var actionDownloadCancellation = CancellationTokenSource.CreateLinkedTokenSource(actionDownloadTimeout.Token, executionContext.CancellationToken)) { @@ -1117,7 +1118,7 @@ private async Task DownloadRepositoryArchive(IExecutionContext executionContext, httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); using (var response = await httpClient.GetAsync(downloadUrl)) { - var requestId = UrlUtil.GetGitHubRequestId(response.Headers); + requestId = UrlUtil.GetGitHubRequestId(response.Headers); if (!string.IsNullOrEmpty(requestId)) { Trace.Info($"Request URL: {downloadUrl} X-GitHub-Request-Id: {requestId} Http Status: {response.StatusCode}"); @@ -1155,7 +1156,7 @@ private async Task DownloadRepositoryArchive(IExecutionContext executionContext, catch (OperationCanceledException ex) when (!executionContext.CancellationToken.IsCancellationRequested && retryCount >= 2) { Trace.Info($"Action download final retry timeout after {timeoutSeconds} seconds."); - throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message}"); + throw new TimeoutException($"Action '{downloadUrl}' download has timed out. Error: {ex.Message} {requestId}"); } catch (ActionNotFoundException) { @@ -1170,11 +1171,11 @@ private async Task DownloadRepositoryArchive(IExecutionContext executionContext, if (actionDownloadTimeout.Token.IsCancellationRequested) { // action download didn't finish within timeout - executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds."); + executionContext.Warning($"Action '{downloadUrl}' didn't finish download within {timeoutSeconds} seconds. {requestId}"); } else { - executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message}"); + executionContext.Warning($"Failed to download action '{downloadUrl}'. Error: {ex.Message} {requestId}"); } } } From 36c66c808363839ea7069bef87fd4572c8bb6c54 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Tue, 3 Sep 2024 17:06:35 -0500 Subject: [PATCH 55/68] Fix issues for composite actions (Run Service flow) (#3446) --- src/Runner.Worker/ExecutionContext.cs | 15 +++-- src/Test/L0/Worker/ExecutionContextL0.cs | 78 +++++++++++++++++++++++- 2 files changed, 86 insertions(+), 7 deletions(-) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 77c145d1ddc..f34a9c54a28 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -83,7 +83,7 @@ public interface IExecutionContext : IRunnerService // Initialize void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token); void CancelToken(); - IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, ActionRunStage stage, Dictionary intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, TimeSpan? timeout = null); + IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, ActionRunStage stage, Dictionary intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, List embeddedIssueCollector = null, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, TimeSpan? timeout = null); IExecutionContext CreateEmbeddedChild(string scopeName, string contextName, Guid embeddedId, ActionRunStage stage, Dictionary intraActionState = null, string siblingScopeName = null); // logging @@ -135,7 +135,6 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext private readonly TimelineRecord _record = new(); private readonly Dictionary _detailRecords = new(); - private readonly List _embeddedIssueCollector; private readonly object _loggerLock = new(); private readonly object _matchersLock = new(); private readonly ExecutionContext _parentExecutionContext; @@ -154,6 +153,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext private CancellationTokenSource _cancellationTokenSource; private TaskCompletionSource _forceCompleted = new(); private bool _throttlingReported = false; + private List _embeddedIssueCollector; // only job level ExecutionContext will track throttling delay. private long _totalThrottlingDelayInMilliseconds = 0; @@ -356,6 +356,7 @@ public IExecutionContext CreateChild( int? recordOrder = null, IPagingLogger logger = null, bool isEmbedded = false, + List embeddedIssueCollector = null, CancellationTokenSource cancellationTokenSource = null, Guid embeddedId = default(Guid), string siblingScopeName = null, @@ -365,6 +366,10 @@ public IExecutionContext CreateChild( var child = new ExecutionContext(this, isEmbedded); child.Initialize(HostContext); + if ((Global.Variables.GetBoolean("RunService.FixEmbeddedIssues") ?? false) && embeddedIssueCollector != null) + { + child._embeddedIssueCollector = embeddedIssueCollector; + } child.Global = Global; child.ScopeName = scopeName; child.ContextName = contextName; @@ -433,7 +438,7 @@ public IExecutionContext CreateEmbeddedChild( Dictionary intraActionState = null, string siblingScopeName = null) { - return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout(), recordOrder: _record.Order); + return Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, contextName, stage, logger: _logger, isEmbedded: true, embeddedIssueCollector: _embeddedIssueCollector, cancellationTokenSource: null, intraActionState: intraActionState, embeddedId: embeddedId, siblingScopeName: siblingScopeName, timeout: GetRemainingTimeout(), recordOrder: _record.Order); } public void Start(string currentOperation = null) @@ -520,7 +525,6 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = Global.StepsResult.Add(stepResult); } - if (Root != this) { // only dispose TokenSource for step level ExecutionContext @@ -837,7 +841,6 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // Actions environment ActionsEnvironment = message.ActionsEnvironment; - // Service container info Global.ServiceContainers = new List(); @@ -1418,7 +1421,7 @@ private static void ResolvePathsInExpressionValuesDictionary(this IExecutionCont { if (key == PipelineTemplateConstants.HostWorkspace) { - // The HostWorkspace context var is excluded so that there is a var that always points to the host path. + // The HostWorkspace context var is excluded so that there is a var that always points to the host path. // This var can be used to translate back from container paths, e.g. in HashFilesFunction, which always runs on the host machine continue; } diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 08abcd09585..bf2a5835b88 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -773,6 +773,82 @@ public void PublishStepTelemetry_EmbeddedStep() [Trait("Level", "L0")] [Trait("Category", "Worker")] public void PublishStepResult_EmbeddedStep() + { + using (TestHostContext hc = CreateTestContext()) + { + // Job request + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); + Guid jobId = Guid.NewGuid(); + string jobName = "some job name"; + var variables = new Dictionary() + { + ["RunService.FixEmbeddedIssues"] = new VariableValue("true"), + }; + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, variables, new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, null); + jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() + { + Alias = Pipelines.PipelineConstants.SelfAlias, + Id = "github", + Version = "sha1" + }); + jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); + + // Mocks + var pagingLogger = new Mock(); + var pagingLogger2 = new Mock(); + var jobServerQueue = new Mock(); + jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.IsAny())); + hc.EnqueueInstance(pagingLogger.Object); + hc.EnqueueInstance(pagingLogger2.Object); + hc.SetSingleton(jobServerQueue.Object); + + // Job context + var jobContext = new Runner.Worker.ExecutionContext(); + jobContext.Initialize(hc); + jobContext.InitializeJob(jobRequest, CancellationToken.None); + jobContext.Start(); + + // Step 1 context + var step1 = jobContext.CreateChild(Guid.NewGuid(), "my_step", "my_step", null, null, ActionRunStage.Main); + step1.Start(); + + // Embedded step 1a context + var embeddedStep1a = step1.CreateEmbeddedChild(null, null, Guid.NewGuid(), ActionRunStage.Main); + embeddedStep1a.Start(); + embeddedStep1a.StepTelemetry.Type = "node16"; + embeddedStep1a.StepTelemetry.Action = "actions/checkout"; + embeddedStep1a.StepTelemetry.Ref = "v2"; + embeddedStep1a.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + embeddedStep1a.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + embeddedStep1a.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }, ExecutionContextLogOptions.Default); + embeddedStep1a.Complete(); + + // Embedded step 1b context + var embeddedStep1b = step1.CreateEmbeddedChild(null, null, Guid.NewGuid(), ActionRunStage.Main); + embeddedStep1b.Start(); + embeddedStep1b.StepTelemetry.Type = "node16"; + embeddedStep1b.StepTelemetry.Action = "actions/checkout"; + embeddedStep1b.StepTelemetry.Ref = "v2"; + embeddedStep1b.AddIssue(new Issue() { Type = IssueType.Error, Message = "error 2" }, ExecutionContextLogOptions.Default); + embeddedStep1b.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning 2" }, ExecutionContextLogOptions.Default); + embeddedStep1b.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice 2" }, ExecutionContextLogOptions.Default); + embeddedStep1b.Complete(); + + step1.Complete(); + + // Assert + Assert.Equal(3, jobContext.Global.StepsResult.Count); + Assert.Equal(0, jobContext.Global.StepsResult[0].Annotations.Count); + Assert.Equal(0, jobContext.Global.StepsResult[1].Annotations.Count); + Assert.Equal(6, jobContext.Global.StepsResult[2].Annotations.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void PublishStepResult_EmbeddedStep_Legacy() { using (TestHostContext hc = CreateTestContext()) { @@ -807,7 +883,7 @@ public void PublishStepResult_EmbeddedStep() ec.InitializeJob(jobRequest, CancellationToken.None); ec.Start(); - var embeddedStep = ec.CreateChild(Guid.NewGuid(), "action_1_pre", "action_1_pre", null, null, ActionRunStage.Main, isEmbedded: true); + var embeddedStep = ec.CreateEmbeddedChild(null, null, Guid.NewGuid(), ActionRunStage.Main); embeddedStep.Start(); embeddedStep.StepTelemetry.Type = "node16"; From 65764d9ddc30e05781b2db1ce647c1433de33582 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 5 Sep 2024 16:12:29 -0400 Subject: [PATCH 56/68] Capature actions_type after resolving alpine container. (#3455) --- src/Runner.Worker/Handlers/NodeScriptActionHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs index 9d412a72314..6090e5be315 100644 --- a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs +++ b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs @@ -93,7 +93,6 @@ public async Task RunAsync(ActionRunStage stage) ExecutionContext.StepTelemetry.HasPreStep = Data.HasPre; ExecutionContext.StepTelemetry.HasPostStep = Data.HasPost; } - ExecutionContext.StepTelemetry.Type = Data.NodeVersion; ArgUtil.NotNullOrEmpty(target, nameof(target)); target = Path.Combine(ActionDirectory, target); @@ -124,6 +123,7 @@ public async Task RunAsync(ActionRunStage stage) Data.NodeVersion = "node20"; } var nodeRuntimeVersion = await StepHost.DetermineNodeRuntimeVersion(ExecutionContext, Data.NodeVersion); + ExecutionContext.StepTelemetry.Type = nodeRuntimeVersion; string file = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), nodeRuntimeVersion, "bin", $"node{IOUtil.ExeExtension}"); // Format the arguments passed to node. From 4c0a43f0e48b21fc23bf7458cd5657863a7a80db Mon Sep 17 00:00:00 2001 From: Luke Tomlinson Date: Thu, 5 Sep 2024 17:08:57 -0400 Subject: [PATCH 57/68] Handle Error Body in Responses from Broker (#3454) --- src/Runner.Common/BrokerServer.cs | 2 +- src/Sdk/RSWebApi/Contracts/BrokerError.cs | 20 +++++++++ src/Sdk/RSWebApi/Contracts/BrokerErrorKind.cs | 10 +++++ src/Sdk/WebApi/WebApi/BrokerHttpClient.cs | 41 +++++++++++++++++-- 4 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 src/Sdk/RSWebApi/Contracts/BrokerError.cs create mode 100644 src/Sdk/RSWebApi/Contracts/BrokerErrorKind.cs diff --git a/src/Runner.Common/BrokerServer.cs b/src/Runner.Common/BrokerServer.cs index 5e1311715c5..4c612e9615e 100644 --- a/src/Runner.Common/BrokerServer.cs +++ b/src/Runner.Common/BrokerServer.cs @@ -92,7 +92,7 @@ public Task ForceRefreshConnection(VssCredentials credentials) public bool ShouldRetryException(Exception ex) { - if (ex is AccessDeniedException ade && ade.ErrorCode == 1) + if (ex is AccessDeniedException ade) { return false; } diff --git a/src/Sdk/RSWebApi/Contracts/BrokerError.cs b/src/Sdk/RSWebApi/Contracts/BrokerError.cs new file mode 100644 index 00000000000..c2e4bfa7b6b --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/BrokerError.cs @@ -0,0 +1,20 @@ +using System.Runtime.Serialization; + +namespace GitHub.Actions.RunService.WebApi +{ + [DataContract] + public class BrokerError + { + [DataMember(Name = "source", EmitDefaultValue = false)] + public string Source { get; set; } + + [DataMember(Name = "errorKind", EmitDefaultValue = false)] + public string ErrorKind { get; set; } + + [DataMember(Name = "statusCode", EmitDefaultValue = false)] + public int StatusCode { get; set; } + + [DataMember(Name = "errorMessage", EmitDefaultValue = false)] + public string Message { get; set; } + } +} diff --git a/src/Sdk/RSWebApi/Contracts/BrokerErrorKind.cs b/src/Sdk/RSWebApi/Contracts/BrokerErrorKind.cs new file mode 100644 index 00000000000..15c423017db --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/BrokerErrorKind.cs @@ -0,0 +1,10 @@ +using System.Runtime.Serialization; + +namespace GitHub.Actions.RunService.WebApi +{ + [DataContract] + public class BrokerErrorKind + { + public const string RunnerVersionTooOld = "RunnerVersionTooOld"; + } +} diff --git a/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs b/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs index e9ad938fb9f..8b67da2e5b9 100644 --- a/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs @@ -103,6 +103,7 @@ public async Task GetRunnerMessageAsync( new HttpMethod("GET"), requestUri: requestUri, queryParameters: queryParams, + readErrorBody: true, cancellationToken: cancellationToken); if (result.IsSuccess) @@ -110,8 +111,21 @@ public async Task GetRunnerMessageAsync( return result.Value; } - // the only time we throw a `Forbidden` exception from Listener /messages is when the runner is - // disable_update and is too old to poll + if (TryParseErrorBody(result.ErrorBody, out BrokerError brokerError)) + { + switch (brokerError.ErrorKind) + { + case BrokerErrorKind.RunnerVersionTooOld: + throw new AccessDeniedException(brokerError.Message) + { + ErrorCode = 1 + }; + default: + break; + } + } + + // temporary back compat if (result.StatusCode == HttpStatusCode.Forbidden) { throw new AccessDeniedException($"{result.Error} Runner version v{runnerVersion} is deprecated and cannot receive messages.") @@ -120,7 +134,7 @@ public async Task GetRunnerMessageAsync( }; } - throw new Exception($"Failed to get job message: {result.Error}"); + throw new Exception($"Failed to get job message. Request to {requestUri} failed with status: {result.StatusCode}. Error message {result.Error}"); } public async Task CreateSessionAsync( @@ -172,5 +186,26 @@ public async Task DeleteSessionAsync( throw new Exception($"Failed to delete broker session: {result.Error}"); } + + private static bool TryParseErrorBody(string errorBody, out BrokerError error) + { + if (!string.IsNullOrEmpty(errorBody)) + { + try + { + error = JsonUtility.FromString(errorBody); + if (error?.Source == "actions-broker-listener") + { + return true; + } + } + catch (Exception) + { + } + } + + error = null; + return false; + } } } From 0b0cb5520d3bf0dc49e7c353df5062ff18ce521e Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Fri, 6 Sep 2024 17:16:17 -0400 Subject: [PATCH 58/68] Add runner or worker to the useragent. (#3457) --- src/Runner.Common/HostContext.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index 06b0cd5892a..46ea7042d8b 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -248,6 +248,7 @@ public HostContext(string hostType, string logFile = null) var currentProcess = Process.GetCurrentProcess(); _userAgents.Add(new ProductInfoHeaderValue("Pid", currentProcess.Id.ToString())); _userAgents.Add(new ProductInfoHeaderValue("CreationTime", Uri.EscapeDataString(DateTime.UtcNow.ToString("O")))); + _userAgents.Add(new ProductInfoHeaderValue($"({hostType})")); } public string GetDirectory(WellKnownDirectory directory) From ddf41af7678a8bc585d9e6b8ab70d941cdb687b2 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Fri, 6 Sep 2024 17:04:17 -0500 Subject: [PATCH 59/68] Cleanup back-compat code for interpreting Run Service status codes (#3456) --- src/Sdk/RSWebApi/RunServiceHttpClient.cs | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index 14bdd2a6379..2a1b0b998ad 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -107,15 +107,6 @@ public async Task GetJobMessageAsync( } } - // Temporary back compat - switch (result.StatusCode) - { - case HttpStatusCode.NotFound: - throw new TaskOrchestrationJobNotFoundException($"Job message not found: {messageId}"); - case HttpStatusCode.Conflict: - throw new TaskOrchestrationJobAlreadyAcquiredException($"Job message already acquired: {messageId}"); - } - if (!string.IsNullOrEmpty(result.ErrorBody)) { throw new Exception($"Failed to get job message: {result.Error}. {Truncate(result.ErrorBody)}"); @@ -171,13 +162,6 @@ public async Task CompleteJobAsync( } } - // Temporary back compat - switch (result.StatusCode) - { - case HttpStatusCode.NotFound: - throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}"); - } - if (!string.IsNullOrEmpty(result.ErrorBody)) { throw new Exception($"Failed to complete job: {result.Error}. {Truncate(result.ErrorBody)}"); @@ -225,13 +209,6 @@ public async Task RenewJobAsync( } } - // Temporary back compat - switch (result.StatusCode) - { - case HttpStatusCode.NotFound: - throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}"); - } - if (!string.IsNullOrEmpty(result.ErrorBody)) { throw new Exception($"Failed to renew job: {result.Error}. {Truncate(result.ErrorBody)}"); From 6d7446a45ebc638a842895d5742d6cf9afa3b66d Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 25 Sep 2024 09:01:53 -0400 Subject: [PATCH 60/68] fix missing default user-agent for jitconfig runner. (#3473) --- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 4 ++-- src/Runner.Common/HostContext.cs | 20 ++++++++++++++------ src/Runner.Listener/Runner.cs | 4 ++++ src/Test/L0/TestHostContext.cs | 5 +++++ 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7eabfb9cfe2..c041cf336a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -75,7 +75,7 @@ jobs: # Upload runner package tar.gz/zip as artifact - name: Publish Artifact if: github.event_name != 'pull_request' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: runner-package-${{ matrix.runtime }} path: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de94bcc4b26..73dc6477759 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -120,7 +120,7 @@ jobs: # Since each package name is unique, so we don't need to put ${{matrix}} info into artifact name - name: Publish Artifact if: github.event_name != 'pull_request' - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: runner-packages path: | @@ -135,7 +135,7 @@ jobs: # Download runner package tar.gz/zip produced by 'build' job - name: Download Artifact - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v4 with: name: runner-packages path: ./ diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index 46ea7042d8b..0b2ae0ae979 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -36,6 +36,7 @@ public interface IHostContext : IDisposable event EventHandler Unloading; void ShutdownRunner(ShutdownReason reason); void WritePerfCounter(string counter); + void LoadDefaultUserAgents(); } public enum StartupType @@ -67,6 +68,7 @@ public sealed class HostContext : EventListener, IObserver, private StartupType _startupType; private string _perfFile; private RunnerWebProxy _webProxy = new(); + private string _hostType = string.Empty; public event EventHandler Unloading; public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token; @@ -78,6 +80,7 @@ public HostContext(string hostType, string logFile = null) { // Validate args. ArgUtil.NotNullOrEmpty(hostType, nameof(hostType)); + _hostType = hostType; _loadContext = AssemblyLoadContext.GetLoadContext(typeof(HostContext).GetTypeInfo().Assembly); _loadContext.Unloading += LoadContext_Unloading; @@ -196,6 +199,16 @@ public HostContext(string hostType, string logFile = null) } } + if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY"))) + { + _trace.Warning($"Runner is running under insecure mode: HTTPS server certificate validation has been turned off by GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY environment variable."); + } + + LoadDefaultUserAgents(); + } + + public void LoadDefaultUserAgents() + { if (string.IsNullOrEmpty(WebProxy.HttpProxyAddress) && string.IsNullOrEmpty(WebProxy.HttpsProxyAddress)) { _trace.Info($"No proxy settings were found based on environmental variables (http_proxy/https_proxy/HTTP_PROXY/HTTPS_PROXY)"); @@ -205,11 +218,6 @@ public HostContext(string hostType, string logFile = null) _userAgents.Add(new ProductInfoHeaderValue("HttpProxyConfigured", bool.TrueString)); } - if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY"))) - { - _trace.Warning($"Runner is running under insecure mode: HTTPS server certificate validation has been turned off by GITHUB_ACTIONS_RUNNER_TLS_NO_VERIFY environment variable."); - } - var credFile = GetConfigFile(WellKnownConfigFile.Credentials); if (File.Exists(credFile)) { @@ -248,7 +256,7 @@ public HostContext(string hostType, string logFile = null) var currentProcess = Process.GetCurrentProcess(); _userAgents.Add(new ProductInfoHeaderValue("Pid", currentProcess.Id.ToString())); _userAgents.Add(new ProductInfoHeaderValue("CreationTime", Uri.EscapeDataString(DateTime.UtcNow.ToString("O")))); - _userAgents.Add(new ProductInfoHeaderValue($"({hostType})")); + _userAgents.Add(new ProductInfoHeaderValue($"({_hostType})")); } public string GetDirectory(WellKnownDirectory directory) diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index e2f638522b8..8649af7354f 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -237,6 +237,10 @@ public async Task ExecuteCommand(CommandSettings command) File.SetAttributes(configFile, File.GetAttributes(configFile) | FileAttributes.Hidden); Trace.Info($"Saved {configContent.Length} bytes to '{configFile}'."); } + + // make sure we have the right user agent data added from the jitconfig + HostContext.LoadDefaultUserAgents(); + VssUtil.InitializeVssClientSettings(HostContext.UserAgents, HostContext.WebProxy); } catch (Exception ex) { diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index c44f13f1c4a..124bcd5bd06 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -370,6 +370,11 @@ private void LoadContext_Unloading(AssemblyLoadContext obj) Unloading(this, null); } } + + public void LoadDefaultUserAgents() + { + return; + } } public class DelayEventArgs : EventArgs From 3696b7d89fd8d900973152a7e5c4b505ec6789d9 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 30 Sep 2024 10:57:08 -0400 Subject: [PATCH 61/68] Create launch httpclient using the right handler and setting. (#3476) --- src/Runner.Common/LaunchServer.cs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Runner.Common/LaunchServer.cs b/src/Runner.Common/LaunchServer.cs index e1b1b0f4f7c..f8584ac5341 100644 --- a/src/Runner.Common/LaunchServer.cs +++ b/src/Runner.Common/LaunchServer.cs @@ -1,11 +1,12 @@ using System; using System.Collections.Generic; -using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Sdk; +using GitHub.Services.Common; using GitHub.Services.Launch.Client; -using GitHub.Services.WebApi; namespace GitHub.Runner.Common { @@ -23,8 +24,21 @@ public sealed class LaunchServer : RunnerService, ILaunchServer public void InitializeLaunchClient(Uri uri, string token) { - var httpMessageHandler = HostContext.CreateHttpClientHandler(); - this._launchClient = new LaunchHttpClient(uri, httpMessageHandler, token, disposeHandler: true); + // Using default 100 timeout + RawClientHttpRequestSettings settings = VssUtil.GetHttpRequestSettings(null); + + // Create retry handler + IEnumerable delegatingHandlers = new List(); + if (settings.MaxRetryRequest > 0) + { + delegatingHandlers = new DelegatingHandler[] { new VssHttpRetryMessageHandler(settings.MaxRetryRequest) }; + } + + // Setup RawHttpMessageHandler without credentials + var httpMessageHandler = new RawHttpMessageHandler(new NoOpCredentials(null), settings); + var pipeline = HttpClientFactory.CreatePipeline(httpMessageHandler, delegatingHandlers); + + this._launchClient = new LaunchHttpClient(uri, pipeline, token, disposeHandler: true); } public Task ResolveActionsDownloadInfoAsync(Guid planId, Guid jobId, ActionReferenceList actionReferenceList, From e292ec220ea757e673106ed013cec2e3ac8f8665 Mon Sep 17 00:00:00 2001 From: Raj R <58966550+rajrku@users.noreply.github.com> Date: Tue, 1 Oct 2024 11:04:48 -0700 Subject: [PATCH 62/68] Adding Snapshot additional mapping tokens (#3468) * Adding Snapshot additional mapping tokens * Lint failure fixes * Lint failure fixes - 2 * Lint failure fixes - 3 * Fixed a few nits * Lint fixes * Removed unncessary white space --- src/Runner.Worker/JobExtension.cs | 2 +- .../PipelineTemplateConstants.cs | 1 + .../PipelineTemplateConverter.cs | 34 +++++++++++++- src/Sdk/DTPipelines/Pipelines/Snapshot.cs | 12 ++++- src/Sdk/DTPipelines/workflow-v1.0.json | 17 +++++++ src/Test/L0/Worker/JobExtensionL0.cs | 46 ++++++++++++++++--- 6 files changed, 103 insertions(+), 9 deletions(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index a36e4beb4d8..e111a77aac5 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -403,7 +403,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel var snapshotOperationProvider = HostContext.GetService(); jobContext.RegisterPostJobStep(new JobExtensionRunner( runAsync: (executionContext, _) => snapshotOperationProvider.CreateSnapshotRequestAsync(executionContext, snapshotRequest), - condition: $"{PipelineTemplateConstants.Success}()", + condition: snapshotRequest.Condition, displayName: $"Create custom image", data: null)); } diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs index a7e90fce334..8d81c7d2d53 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs @@ -30,6 +30,7 @@ public sealed class PipelineTemplateConstants public const String If = "if"; public const String Image = "image"; public const String ImageName = "image-name"; + public const String CustomImageVersion = "version"; public const String Include = "include"; public const String Inputs = "inputs"; public const String Job = "job"; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 9d2c0bdca7a..40f6a13345f 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.ComponentModel; +using System.Globalization; using System.Linq; using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.Expressions2.Sdk; @@ -349,6 +350,10 @@ internal static List> ConvertToJobServiceCont internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, TemplateToken token) { string imageName = null; + string version = "1.*"; + string versionString = string.Empty; + var condition = $"{PipelineTemplateConstants.Success}()"; + if (token is StringToken snapshotStringLiteral) { imageName = snapshotStringLiteral.Value; @@ -359,11 +364,19 @@ internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, Te foreach (var snapshotPropertyPair in snapshotMapping) { var propertyName = snapshotPropertyPair.Key.AssertString($"{PipelineTemplateConstants.Snapshot} key"); + var propertyValue = snapshotPropertyPair.Value; switch (propertyName.Value) { case PipelineTemplateConstants.ImageName: imageName = snapshotPropertyPair.Value.AssertString($"{PipelineTemplateConstants.Snapshot} {propertyName}").Value; break; + case PipelineTemplateConstants.If: + condition = ConvertToIfCondition(context, propertyValue, false); + break; + case PipelineTemplateConstants.CustomImageVersion: + versionString = propertyValue.AssertString($"job {PipelineTemplateConstants.Snapshot} {PipelineTemplateConstants.CustomImageVersion}").Value; + version = IsSnapshotImageVersionValid(versionString) ? versionString : null; + break; default: propertyName.AssertUnexpectedValue($"{PipelineTemplateConstants.Snapshot} key"); break; @@ -376,7 +389,26 @@ internal static Snapshot ConvertToJobSnapshotRequest(TemplateContext context, Te return null; } - return new Snapshot(imageName); + return new Snapshot(imageName) + { + Condition = condition, + Version = version + }; + } + + private static bool IsSnapshotImageVersionValid(string versionString) + { + var versionSegments = versionString.Split("."); + + if (versionSegments.Length != 2 || + !versionSegments[1].Equals("*") || + !Int32.TryParse(versionSegments[0], NumberStyles.None, CultureInfo.InvariantCulture, result: out int parsedMajor) || + parsedMajor < 0) + { + return false; + } + + return true; } private static ActionStep ConvertToStep( diff --git a/src/Sdk/DTPipelines/Pipelines/Snapshot.cs b/src/Sdk/DTPipelines/Pipelines/Snapshot.cs index 60f8da04f4f..c1a05674aea 100644 --- a/src/Sdk/DTPipelines/Pipelines/Snapshot.cs +++ b/src/Sdk/DTPipelines/Pipelines/Snapshot.cs @@ -1,17 +1,27 @@ using System; using System.Runtime.Serialization; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ObjectTemplating; namespace GitHub.DistributedTask.Pipelines { [DataContract] public class Snapshot { - public Snapshot(string imageName) + public Snapshot(string imageName, string condition = null, string version = null) { ImageName = imageName; + Condition = condition ?? $"{PipelineTemplateConstants.Success}()"; + Version = version ?? "1.*"; } [DataMember(EmitDefaultValue = false)] public String ImageName { get; set; } + + [DataMember(EmitDefaultValue = false)] + public String Condition { get; set; } + + [DataMember(EmitDefaultValue = false)] + public String Version { get; set; } } } diff --git a/src/Sdk/DTPipelines/workflow-v1.0.json b/src/Sdk/DTPipelines/workflow-v1.0.json index a3837edff02..ec09cfe58b7 100644 --- a/src/Sdk/DTPipelines/workflow-v1.0.json +++ b/src/Sdk/DTPipelines/workflow-v1.0.json @@ -169,11 +169,28 @@ "image-name": { "type": "non-empty-string", "required": true + }, + "if": "snapshot-if", + "version": { + "type": "non-empty-string", + "required": false } } } }, + "snapshot-if": { + "context": [ + "github", + "inputs", + "vars", + "needs", + "strategy", + "matrix" + ], + "string": {} + }, + "runs-on": { "context": [ "github", diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index 4f5834fbe69..5db32d47a30 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -506,7 +506,27 @@ public Task EnsureSnapshotPostJobStepForMappingToken() return EnsureSnapshotPostJobStepForToken(mappingToken, snapshot); } - private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken, Pipelines.Snapshot expectedSnapshot) + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public Task EnsureSnapshotPostJobStepForMappingToken_WithIf_Is_False() + { + var snapshot = new Pipelines.Snapshot("TestImageNameFromMappingToken", condition: $"{PipelineTemplateConstants.Success}() && 1==0", version: "2.*"); + var imageNameValueStringToken = new StringToken(null, null, null, snapshot.ImageName); + var condition = new StringToken(null, null, null, snapshot.Condition); + var version = new StringToken(null, null, null, snapshot.Version); + + var mappingToken = new MappingToken(null, null, null) + { + { new StringToken(null,null,null, PipelineTemplateConstants.ImageName), imageNameValueStringToken }, + { new StringToken(null,null,null, PipelineTemplateConstants.If), condition }, + { new StringToken(null,null,null, PipelineTemplateConstants.CustomImageVersion), version } + }; + + return EnsureSnapshotPostJobStepForToken(mappingToken, snapshot, skipSnapshotStep: true); + } + + private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken, Pipelines.Snapshot expectedSnapshot, bool skipSnapshotStep = false) { using (TestHostContext hc = CreateTestContext()) { @@ -524,14 +544,28 @@ private async Task EnsureSnapshotPostJobStepForToken(TemplateToken snapshotToken Assert.Equal(1, postJobSteps.Count); var snapshotStep = postJobSteps.First(); + _jobEc.JobSteps.Enqueue(snapshotStep); + + var _stepsRunner = new StepsRunner(); + _stepsRunner.Initialize(hc); + await _stepsRunner.RunAsync(_jobEc); + Assert.Equal("Create custom image", snapshotStep.DisplayName); - Assert.Equal($"{PipelineTemplateConstants.Success}()", snapshotStep.Condition); + Assert.Equal(expectedSnapshot.Condition ?? $"{PipelineTemplateConstants.Success}()", snapshotStep.Condition); // Run the mock snapshot step, so we can verify it was executed with the expected snapshot object. - await snapshotStep.RunAsync(); - - Assert.NotNull(_requestedSnapshot); - Assert.Equal(expectedSnapshot.ImageName, _requestedSnapshot.ImageName); + // await snapshotStep.RunAsync(); + if (skipSnapshotStep) + { + Assert.Null(_requestedSnapshot); + } + else + { + Assert.NotNull(_requestedSnapshot); + Assert.Equal(expectedSnapshot.ImageName, _requestedSnapshot.ImageName); + Assert.Equal(expectedSnapshot.Condition ?? $"{PipelineTemplateConstants.Success}()", _requestedSnapshot.Condition); + Assert.Equal(expectedSnapshot.Version ?? "1.*", _requestedSnapshot.Version); + } } } } From 149123c2322b61d9a497b49cdce6784b23ebfb5d Mon Sep 17 00:00:00 2001 From: eric sciple Date: Thu, 3 Oct 2024 13:38:35 -0500 Subject: [PATCH 63/68] Prepare v2.320.0 (#3484) --- releaseNote.md | 17 ++++++++++++----- src/runnerversion | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 9753b6d2311..70d85af2dcd 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,10 +1,17 @@ ## What's Changed -- .NET 8 OS compatibility test https://github.com/actions/runner/pull/3422 -- Ignore ssl cert on websocket client https://github.com/actions/runner/pull/3423 -- Revert "Bump runner to dotnet 8" https://github.com/actions/runner/pull/3412 - -**Full Changelog**: https://github.com/actions/runner/compare/v2.318.0...v2.319.0 +- Adding Snapshot additional mapping tokens https://github.com/actions/runner/pull/3468 +- Create launch httpclient using the right handler and setting https://github.com/actions/runner/pull/3476 +- Fix missing default user-agent for jitconfig runner https://github.com/actions/runner/pull/3473 +- Cleanup back-compat code for interpreting Run Service status codes https://github.com/actions/runner/pull/3456 +- Add runner or worker to the useragent https://github.com/actions/runner/pull/3457 +- Handle Error Body in Responses from Broker https://github.com/actions/runner/pull/3454 +- Fix issues for composite actions (Run Service flow) https://github.com/actions/runner/pull/3446 +- Trace GitHub RequestId to log https://github.com/actions/runner/pull/3442 +- Add `jq`, `git`, `unzip` and `curl` to default packages installed https://github.com/actions/runner/pull/3056 +- Add pid to user-agent and session owner https://github.com/actions/runner/pull/3432 + +**Full Changelog**: https://github.com/actions/runner/compare/v2.319.1...v2.320.0 _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. diff --git a/src/runnerversion b/src/runnerversion index 4c47f6f2f86..8084ad3118b 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.319.1 +2.320.0 From dcc64fead29079c91dd13f00efd474a70ae2a064 Mon Sep 17 00:00:00 2001 From: eric sciple Date: Thu, 3 Oct 2024 14:43:10 -0500 Subject: [PATCH 64/68] Fix release workflow to use distinct artifact names (#3485) --- .github/workflows/release.yml | 37 +++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 73dc6477759..f4571b6cb8f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -117,12 +117,11 @@ jobs: working-directory: _package # Upload runner package tar.gz/zip as artifact. - # Since each package name is unique, so we don't need to put ${{matrix}} info into artifact name - name: Publish Artifact if: github.event_name != 'pull_request' uses: actions/upload-artifact@v4 with: - name: runner-packages + name: runner-packages-${{ matrix.runtime }} path: | _package @@ -134,10 +133,40 @@ jobs: - uses: actions/checkout@v3 # Download runner package tar.gz/zip produced by 'build' job - - name: Download Artifact + - name: Download Artifact (win-x64) uses: actions/download-artifact@v4 with: - name: runner-packages + name: runner-packages-win-x64 + path: ./ + - name: Download Artifact (win-arm64) + uses: actions/download-artifact@v4 + with: + name: runner-packages-win-arm64 + path: ./ + - name: Download Artifact (osx-x64) + uses: actions/download-artifact@v4 + with: + name: runner-packages-osx-x64 + path: ./ + - name: Download Artifact (osx-arm64) + uses: actions/download-artifact@v4 + with: + name: runner-packages-osx-arm64 + path: ./ + - name: Download Artifact (linux-x64) + uses: actions/download-artifact@v4 + with: + name: runner-packages-linux-x64 + path: ./ + - name: Download Artifact (linux-arm) + uses: actions/download-artifact@v4 + with: + name: runner-packages-linux-arm + path: ./ + - name: Download Artifact (linux-arm64) + uses: actions/download-artifact@v4 + with: + name: runner-packages-linux-arm64 path: ./ # Create ReleaseNote file From aa0ee2bf64d28343ca72dee7f7b4ccbd12567e7d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:12:31 -0400 Subject: [PATCH 65/68] Upgrade dotnet sdk to v6.0.425 (#3433) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .devcontainer/devcontainer.json | 2 +- src/dev.sh | 2 +- src/global.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 032387bec99..c3255f7e5d6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,7 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:1": {}, "ghcr.io/devcontainers/features/dotnet": { - "version": "6.0.421" + "version": "6.0.425" }, "ghcr.io/devcontainers/features/node:1": { "version": "16" diff --git a/src/dev.sh b/src/dev.sh index 71c80637c02..8536aae665f 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -17,7 +17,7 @@ LAYOUT_DIR="$SCRIPT_DIR/../_layout" DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x" PACKAGE_DIR="$SCRIPT_DIR/../_package" DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk" -DOTNETSDK_VERSION="6.0.421" +DOTNETSDK_VERSION="6.0.425" DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION" DOTNET8SDK_VERSION="8.0.303" DOTNET8SDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNET8SDK_VERSION" diff --git a/src/global.json b/src/global.json index e7028fe0dd4..e60945a9b0a 100644 --- a/src/global.json +++ b/src/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "6.0.421" + "version": "6.0.425" } } From 4d8402c2605c34a8b35a31a489fb1bf03c3a3c5c Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Tue, 8 Oct 2024 15:52:48 -0400 Subject: [PATCH 66/68] add ref and type to job completion in run service (#3492) * add ref and type to job completion in run service * lint --- src/Runner.Worker/ExecutionContext.cs | 2 ++ src/Sdk/RSWebApi/Contracts/StepResult.cs | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index f34a9c54a28..6d53c5438d9 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -508,6 +508,8 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = Status = _record.State, Number = _record.Order, Name = _record.Name, + Ref = StepTelemetry?.Ref, + Type = StepTelemetry?.Type, StartedAt = _record.StartTime, CompletedAt = _record.FinishTime, Annotations = new List() diff --git a/src/Sdk/RSWebApi/Contracts/StepResult.cs b/src/Sdk/RSWebApi/Contracts/StepResult.cs index 1da4a2f9797..85886f31f2b 100644 --- a/src/Sdk/RSWebApi/Contracts/StepResult.cs +++ b/src/Sdk/RSWebApi/Contracts/StepResult.cs @@ -19,6 +19,12 @@ public class StepResult [DataMember(Name = "name", EmitDefaultValue = false)] public string Name { get; set; } + [DataMember(Name = "ref", EmitDefaultValue = false)] + public string Ref { get; set; } + + [DataMember(Name = "type", EmitDefaultValue = false)] + public string Type { get; set; } + [DataMember(Name = "status")] public TimelineRecordState? Status { get; set; } From 9b3b554758c26bf2caa8121278ec94a6f0d97de5 Mon Sep 17 00:00:00 2001 From: Luke Tomlinson Date: Wed, 9 Oct 2024 11:07:48 -0400 Subject: [PATCH 67/68] Remove Broker Migration Message logging (#3493) --- src/Runner.Listener/MessageListener.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 6335667267e..6a5c9368edc 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -264,8 +264,6 @@ public async Task GetNextMessageAsync(CancellationToken token) if (message != null && message.MessageType == BrokerMigrationMessage.MessageType) { - Trace.Info("BrokerMigration message received. Polling Broker for messages..."); - var migrationMessage = JsonUtility.FromString(message.Body); await _brokerServer.UpdateConnectionIfNeeded(migrationMessage.BrokerBaseUrl, _creds); From 2f5e87322202d55044e6169d3f8cd6e8f3300d01 Mon Sep 17 00:00:00 2001 From: mattrwi <107269655+mattrwi@users.noreply.github.com> Date: Thu, 10 Oct 2024 05:40:29 -0500 Subject: [PATCH 68/68] Update runnerversion --- src/runnerversion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runnerversion b/src/runnerversion index a611af2d38d..8b100b0cc89 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.320.0 \ No newline at end of file +3.320.0