From 20edd4658d4036024102f42222378fe65a488cb9 Mon Sep 17 00:00:00 2001 From: Eric Sciple Date: Fri, 3 May 2019 11:26:38 -0400 Subject: [PATCH] script runner --- src/Agent.Listener/Agent.Listener.csproj | 2 +- src/Agent.PluginHost/Agent.PluginHost.csproj | 2 +- src/Agent.Plugins/Agent.Plugins.csproj | 2 +- src/Agent.Sdk/Agent.Sdk.csproj | 2 +- src/Agent.Worker/ActionCommandManager.cs | 41 +++-- src/Agent.Worker/ActionsContext.cs | 77 +++++++++ src/Agent.Worker/Agent.Worker.csproj | 2 +- src/Agent.Worker/ExecutionContext.cs | 69 ++++++-- src/Agent.Worker/ExpressionManager.cs | 42 ++++- src/Agent.Worker/Handlers/Handler.cs | 2 +- src/Agent.Worker/Handlers/HandlerFactory.cs | 5 - .../{ShellHandler.cs => ScriptHandler.cs} | 153 +++++++----------- src/Agent.Worker/JobExtension.cs | 44 +++-- src/Agent.Worker/ScriptRunner.cs | 83 ++++++++++ src/Agent.Worker/TaskManager.cs | 45 ------ src/Agent.Worker/WorkerUtilties.cs | 1 + ...crosoft.VisualStudio.Services.Agent.csproj | 2 +- src/Test/Test.csproj | 2 +- 18 files changed, 375 insertions(+), 201 deletions(-) create mode 100644 src/Agent.Worker/ActionsContext.cs rename src/Agent.Worker/Handlers/{ShellHandler.cs => ScriptHandler.cs} (51%) create mode 100644 src/Agent.Worker/ScriptRunner.cs diff --git a/src/Agent.Listener/Agent.Listener.csproj b/src/Agent.Listener/Agent.Listener.csproj index 7f592b6bd40..f88ed8d8ef3 100644 --- a/src/Agent.Listener/Agent.Listener.csproj +++ b/src/Agent.Listener/Agent.Listener.csproj @@ -21,7 +21,7 @@ - + diff --git a/src/Agent.PluginHost/Agent.PluginHost.csproj b/src/Agent.PluginHost/Agent.PluginHost.csproj index ce19af99b58..c3695d0f88d 100644 --- a/src/Agent.PluginHost/Agent.PluginHost.csproj +++ b/src/Agent.PluginHost/Agent.PluginHost.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/Agent.Plugins/Agent.Plugins.csproj b/src/Agent.Plugins/Agent.Plugins.csproj index d852d50bf8c..889278f6da7 100644 --- a/src/Agent.Plugins/Agent.Plugins.csproj +++ b/src/Agent.Plugins/Agent.Plugins.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/Agent.Sdk/Agent.Sdk.csproj b/src/Agent.Sdk/Agent.Sdk.csproj index f893c218f2d..1a3377e61f9 100644 --- a/src/Agent.Sdk/Agent.Sdk.csproj +++ b/src/Agent.Sdk/Agent.Sdk.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/Agent.Worker/ActionCommandManager.cs b/src/Agent.Worker/ActionCommandManager.cs index 168a6c9f840..a7be5ab1510 100644 --- a/src/Agent.Worker/ActionCommandManager.cs +++ b/src/Agent.Worker/ActionCommandManager.cs @@ -63,7 +63,8 @@ public bool TryProcessCommand(IExecutionContext context, string input) { if (string.Equals(actionCommand.Command, _stopCommand, StringComparison.OrdinalIgnoreCase)) { - context.Output($"Stop process further commands till '##[{actionCommand.Data}]' come in."); + context.Output(input); + context.Output($"{WellKnownTags.Debug}Paused processing commands until '##[{actionCommand.Data}]' is received"); _stopToken = actionCommand.Data; _stopProcessCommand = true; _registeredCommands.Add(_stopToken); @@ -72,7 +73,8 @@ public bool TryProcessCommand(IExecutionContext context, string input) else if (!string.IsNullOrEmpty(_stopToken) && string.Equals(actionCommand.Command, _stopToken, StringComparison.OrdinalIgnoreCase)) { - context.Output($"Resume process further commands."); + context.Output(input); + context.Output($"{WellKnownTags.Debug}Resume processing commands"); _registeredCommands.Remove(_stopToken); _stopProcessCommand = false; _stopToken = null; @@ -80,19 +82,24 @@ public bool TryProcessCommand(IExecutionContext context, string input) } else if (_commandExtensions.TryGetValue(actionCommand.Command, out IActionCommandExtension extension)) { + bool omitEcho; try { - extension.ProcessCommand(context, actionCommand); + extension.ProcessCommand(context, input, actionCommand, out omitEcho); } catch (Exception ex) { + omitEcho = true; + context.Output(input); context.Error(StringUtil.Loc("CommandProcessFailed", input)); context.Error(ex); context.CommandResult = TaskResult.Failed; } - finally + + if (!omitEcho) { - context.Debug($"Processed: {input}"); + context.Output(input); + context.Output($"{WellKnownTags.Debug}Processed command"); } } @@ -111,7 +118,7 @@ public interface IActionCommandExtension : IExtension { string Command { get; } - void ProcessCommand(IExecutionContext context, ActionCommand command); + void ProcessCommand(IExecutionContext context, string line, ActionCommand command, out bool omitEcho); } public sealed class SetEnvCommandExtension : AgentService, IActionCommandExtension @@ -120,7 +127,7 @@ public sealed class SetEnvCommandExtension : AgentService, IActionCommandExtensi public Type ExtensionType => typeof(IActionCommandExtension); - public void ProcessCommand(IExecutionContext context, ActionCommand command) + public void ProcessCommand(IExecutionContext context, string line, ActionCommand command, out bool omitEcho) { if (!command.Properties.TryGetValue(SetEnvCommandProperties.Name, out string envName) || string.IsNullOrEmpty(envName)) { @@ -128,6 +135,9 @@ public void ProcessCommand(IExecutionContext context, ActionCommand command) } context.SetVariable(envName, command.Data); + context.Output(line); + context.Output($"{WellKnownTags.Debug}{envName}='{command.Data}'"); + omitEcho = true; } private static class SetEnvCommandProperties @@ -142,14 +152,17 @@ public sealed class SetOutputCommandExtension : AgentService, IActionCommandExte public Type ExtensionType => typeof(IActionCommandExtension); - public void ProcessCommand(IExecutionContext context, ActionCommand command) + public void ProcessCommand(IExecutionContext context, string line, ActionCommand command, out bool omitEcho) { if (!command.Properties.TryGetValue(SetOutputCommandProperties.Name, out string outputName) || string.IsNullOrEmpty(outputName)) { throw new Exception(StringUtil.Loc("MissingOutputName")); } - context.SetOutput(outputName, command.Data); + context.SetOutput(outputName, command.Data, out var reference); + context.Output(line); + context.Output($"{WellKnownTags.Debug}{reference}='{command.Data}'"); + omitEcho = true; } private static class SetOutputCommandProperties @@ -164,14 +177,14 @@ public sealed class SetSecretCommandExtension : AgentService, IActionCommandExte public Type ExtensionType => typeof(IActionCommandExtension); - public void ProcessCommand(IExecutionContext context, ActionCommand command) + public void ProcessCommand(IExecutionContext context, string line, ActionCommand command, out bool omitEcho) { if (!command.Properties.TryGetValue(SetSecretCommandProperties.Name, out string secretName) || string.IsNullOrEmpty(secretName)) { throw new Exception(StringUtil.Loc("MissingSecretName")); } - context.SetOutput(secretName, command.Data, isSecret: true); + throw new NotSupportedException("Not supported yet"); } private static class SetSecretCommandProperties @@ -186,11 +199,12 @@ public sealed class AddPathCommandExtension : AgentService, IActionCommandExtens public Type ExtensionType => typeof(IActionCommandExtension); - public void ProcessCommand(IExecutionContext context, ActionCommand command) + public void ProcessCommand(IExecutionContext context, string line, ActionCommand command, out bool omitEcho) { ArgUtil.NotNullOrEmpty(command.Data, "path"); context.PrependPath.RemoveAll(x => string.Equals(x, command.Data, StringComparison.CurrentCulture)); context.PrependPath.Add(command.Data); + omitEcho = false; } } @@ -215,8 +229,9 @@ public abstract class IssueCommandExtension : AgentService, IActionCommandExtens public Type ExtensionType => typeof(IActionCommandExtension); - public void ProcessCommand(IExecutionContext context, ActionCommand command) + public void ProcessCommand(IExecutionContext context, string inputLine, ActionCommand command, out bool omitEcho) { + omitEcho = false; command.Properties.TryGetValue(IssueCommandProperties.File, out string file); command.Properties.TryGetValue(IssueCommandProperties.Line, out string line); command.Properties.TryGetValue(IssueCommandProperties.Column, out string column); diff --git a/src/Agent.Worker/ActionsContext.cs b/src/Agent.Worker/ActionsContext.cs new file mode 100644 index 00000000000..8a228ad56b0 --- /dev/null +++ b/src/Agent.Worker/ActionsContext.cs @@ -0,0 +1,77 @@ +using Microsoft.TeamFoundation.DistributedTask.WebApi; +using Microsoft.VisualStudio.Services.Agent.Util; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Text.RegularExpressions; + +namespace Microsoft.VisualStudio.Services.Agent.Worker +{ + public sealed class ActionsContext : IReadOnlyDictionary + { + private static readonly Regex _propertyRegex = new Regex("^[a-zA-Z_][a-zA-Z0-9_]*$", RegexOptions.Compiled); + private readonly Dictionary _dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public Int32 Count => _dictionary.Count; + + public IEnumerable Keys => _dictionary.Keys; + + public IEnumerable Values => _dictionary.Values; + + public object this[string key] => _dictionary[key]; + + public Boolean ContainsKey(string key) => _dictionary.ContainsKey(key); + + public IEnumerator> GetEnumerator() => _dictionary.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => _dictionary.GetEnumerator(); + + public Boolean TryGetValue( + string key, + out object value) + { + return _dictionary.TryGetValue(key, out value); + } + + public void SetOutput( + string stepName, + string key, + string value, + out string reference) + { + var action = GetAction(stepName); + var outputs = action["outputs"] as Dictionary; + outputs[key] = value; + if (_propertyRegex.IsMatch(key)) + { + reference = $"actions.{stepName}.outputs.{key}"; + } + else + { + reference = $"actions['{stepName}']['outputs']['{key}']"; + } + } + + public void SetResult( + string stepName, + string result) + { + var action = GetAction(stepName); + action["result"] = result; + } + + private Dictionary GetAction(string stepName) + { + if (_dictionary.TryGetValue(stepName, out var actionObject)) + { + return actionObject as Dictionary; + } + + var action = new Dictionary(StringComparer.OrdinalIgnoreCase); + action.Add("result", null); + action.Add("outputs", new Dictionary(StringComparer.OrdinalIgnoreCase)); + _dictionary.Add(stepName, action); + return action; + } + } +} diff --git a/src/Agent.Worker/Agent.Worker.csproj b/src/Agent.Worker/Agent.Worker.csproj index f7917a76c58..f93af1f3adc 100644 --- a/src/Agent.Worker/Agent.Worker.csproj +++ b/src/Agent.Worker/Agent.Worker.csproj @@ -16,7 +16,7 @@ - + diff --git a/src/Agent.Worker/ExecutionContext.cs b/src/Agent.Worker/ExecutionContext.cs index 768dcd92e55..f353da903db 100644 --- a/src/Agent.Worker/ExecutionContext.cs +++ b/src/Agent.Worker/ExecutionContext.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Collections.Specialized; +using System.Globalization; using System.IO; using System.Linq; using System.Threading; @@ -10,6 +11,7 @@ using Microsoft.VisualStudio.Services.WebApi; using Microsoft.TeamFoundation.DistributedTask.WebApi; using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines; +using ObjectTemplating = Microsoft.TeamFoundation.DistributedTask.ObjectTemplating; using Microsoft.VisualStudio.Services.Agent.Util; namespace Microsoft.VisualStudio.Services.Agent.Worker @@ -38,6 +40,7 @@ public interface IExecutionContext : IAgentService Variables Variables { get; } Variables TaskVariables { get; } HashSet OutputVariables { get; } + IDictionary ExpressionValues { get; } List AsyncCommands { get; } List PrependPath { get; } ContainerInfo Container { get; } @@ -57,7 +60,7 @@ public interface IExecutionContext : IAgentService void Start(string currentOperation = null); TaskResult Complete(TaskResult? result = null, string currentOperation = null, string resultCode = null); void SetVariable(string name, string value, bool isSecret = false, bool isOutput = false, bool isFilePath = false); - void SetOutput(string name, string value, bool isSecret = false); + void SetOutput(string name, string value, out string reference); void SetTimeout(TimeSpan? timeout); void AddIssue(Issue issue); void Progress(int percentage, string currentOperation = null); @@ -103,6 +106,7 @@ public sealed class ExecutionContext : AgentService, IExecutionContext public Variables Variables { get; private set; } public Variables TaskVariables { get; private set; } public HashSet OutputVariables => _outputvariables; + public IDictionary ExpressionValues { get; private set; } public bool WriteDebug { get; private set; } public List PrependPath { get; private set; } public ContainerInfo Container { get; private set; } @@ -175,6 +179,7 @@ public IExecutionContext CreateChild(Guid recordId, string displayName, string r child.Containers = Containers; child.SecureFiles = SecureFiles; child.TaskVariables = taskVariables; + child.ExpressionValues = ExpressionValues; child._cancellationTokenSource = new CancellationTokenSource(); child.WriteDebug = WriteDebug; child._parentExecutionContext = this; @@ -270,26 +275,14 @@ public void SetVariable(string name, string value, bool isSecret = false, bool i } } - public void SetOutput(string name, string value, bool isSecret = false) + public void SetOutput(string name, string value, out string reference) { ArgUtil.NotNullOrEmpty(name, nameof(name)); - if (OutputVariables.Contains(name)) - { - _record.Variables[name] = new VariableValue() - { - Value = value, - IsSecret = isSecret - }; - _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record); + // todo: restrict multiline? - ArgUtil.NotNullOrEmpty(_record.RefName, nameof(_record.RefName)); - Variables.Set($"{_record.RefName}.{name}", value, secret: isSecret); - } - else - { - throw new InvalidOperationException($"'{name}' is not defined as an output value."); - } + var actionsContext = ExpressionValues["actions"] as ActionsContext; + actionsContext.SetOutput(_record.RefName, name, value, out reference); } public void SetTimeout(TimeSpan? timeout) @@ -426,6 +419,17 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation List warnings; Variables = new Variables(HostContext, message.Variables, out warnings); + // Expression values + ExpressionValues = new Dictionary(); + if (message.ExpressionValues?.Count > 0) + { + foreach (var pair in message.ExpressionValues) + { + ExpressionValues[pair.Key] = pair.Value; + } + } + ExpressionValues["actions"] = new ActionsContext(); + // Prepend Path PrependPath = new List(); @@ -730,6 +734,37 @@ public static void Debug(this IExecutionContext context, string message) context.Write(WellKnownTags.Debug, message); } } + + public static ObjectTemplating.ITraceWriter ToTemplateTraceWriter(this IExecutionContext context) + { + return new TemplateTraceWriter(context); + } + } + + internal sealed class TemplateTraceWriter : ObjectTemplating.ITraceWriter + { + private readonly IExecutionContext _executionContext; + + internal TemplateTraceWriter(IExecutionContext executionContext) + { + _executionContext = executionContext; + } + + public void Error(string format, params Object[] args) + { + _executionContext.Error(string.Format(CultureInfo.CurrentCulture, format, args)); + } + + public void Info(string format, params Object[] args) + { + _executionContext.Output(string.Format(CultureInfo.CurrentCulture, $"{WellKnownTags.Debug}{format}", args)); + } + + public void Verbose(string format, params Object[] args) + { + // // todo: switch to verbose? how to set system.debug? + // _executionContext.Output(string.Format(CultureInfo.CurrentCulture, $"{WellKnownTags.Debug}{format}", args)); + } } public static class WellKnownTags diff --git a/src/Agent.Worker/ExpressionManager.cs b/src/Agent.Worker/ExpressionManager.cs index 740210f5aad..e19b7bb1464 100644 --- a/src/Agent.Worker/ExpressionManager.cs +++ b/src/Agent.Worker/ExpressionManager.cs @@ -1,17 +1,19 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.TeamFoundation.DistributedTask.Expressions; using Microsoft.TeamFoundation.DistributedTask.WebApi; using Microsoft.VisualStudio.Services.Agent.Util; -using Microsoft.TeamFoundation.DistributedTask.Expressions; -using System.Text; +using ObjectTemplating = Microsoft.TeamFoundation.DistributedTask.ObjectTemplating; namespace Microsoft.VisualStudio.Services.Agent.Worker { [ServiceLocator(Default = typeof(ExpressionManager))] public interface IExpressionManager : IAgentService { - IExpressionNode Parse(IExecutionContext context, string condition); + IExpressionNode Parse(IExecutionContext context, string condition, bool legacy); ConditionResult Evaluate(IExecutionContext context, IExpressionNode tree, bool hostTracingOnly = false); } @@ -21,15 +23,23 @@ public sealed class ExpressionManager : AgentService, IExpressionManager public static IExpressionNode Succeeded = new SucceededNode(); public static IExpressionNode SucceededOrFailed = new SucceededOrFailedNode(); - public IExpressionNode Parse(IExecutionContext executionContext, string condition) + public IExpressionNode Parse(IExecutionContext executionContext, string condition, bool legacy) { ArgUtil.NotNull(executionContext, nameof(executionContext)); var expressionTrace = new TraceWriter(Trace, executionContext); var parser = new ExpressionParser(); - var namedValues = new INamedValueInfo[] + var namedValues = default(INamedValueInfo[]); + if (legacy) { - new NamedValueInfo(name: Constants.Expressions.Variables), - }; + namedValues = new INamedValueInfo[] + { + new NamedValueInfo(name: Constants.Expressions.Variables), + }; + } + else + { + namedValues = executionContext.ExpressionValues.Keys.Select(x => new NamedValueInfo(x)).ToArray(); + } var functions = new IFunctionInfo[] { new FunctionInfo(name: Constants.Expressions.Always, minParameters: 0, maxParameters: 0), @@ -49,7 +59,13 @@ public ConditionResult Evaluate(IExecutionContext executionContext, IExpressionN ConditionResult result = new ConditionResult(); var expressionTrace = new TraceWriter(Trace, hostTracingOnly ? null : executionContext); - result.Value = tree.Evaluate(trace: expressionTrace, secretMasker: HostContext.SecretMasker, state: executionContext); + var evaluationOptions = new EvaluationOptions + { + Converters = new ObjectTemplating.TemplateContext().ExpressionConverters, + UseCollectionInterfaces = true, + }; + + result.Value = tree.Evaluate(trace: expressionTrace, secretMasker: HostContext.SecretMasker, state: executionContext, options: evaluationOptions); result.Trace = expressionTrace.Trace; return result; @@ -189,6 +205,16 @@ public bool TryGetValue(string key, out object value) // IEnumerable members IEnumerator IEnumerable.GetEnumerator() => throw new NotSupportedException(); } + + private sealed class ContextValueNode : NamedValueNode + { + protected override Object EvaluateCore(EvaluationContext evaluationContext) + { + var jobContext = evaluationContext.State as IExecutionContext; + ArgUtil.NotNull(jobContext, nameof(jobContext)); + return jobContext.ExpressionValues[Name]; + } + } } public class ConditionResult diff --git a/src/Agent.Worker/Handlers/Handler.cs b/src/Agent.Worker/Handlers/Handler.cs index e77ef4695ba..5eec835e8c1 100644 --- a/src/Agent.Worker/Handlers/Handler.cs +++ b/src/Agent.Worker/Handlers/Handler.cs @@ -50,7 +50,7 @@ public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); CommandManager = hostContext.GetService(); - ActionCommandManager = hostContext.GetService(); + ActionCommandManager = hostContext.CreateService(); } protected void AddEndpointsToEnvironment() diff --git a/src/Agent.Worker/Handlers/HandlerFactory.cs b/src/Agent.Worker/Handlers/HandlerFactory.cs index 31b5103c003..3b1678a0dc8 100644 --- a/src/Agent.Worker/Handlers/HandlerFactory.cs +++ b/src/Agent.Worker/Handlers/HandlerFactory.cs @@ -118,11 +118,6 @@ public IHandler Create( handler = HostContext.CreateService(); (handler as INodeScriptActionHandler).Data = data as NodeScriptActionHandlerData; } - else if (data is ShellHandlerData) - { - handler = HostContext.CreateService(); - (handler as IShellHandler).Data = data as ShellHandlerData; - } else { // This should never happen. diff --git a/src/Agent.Worker/Handlers/ShellHandler.cs b/src/Agent.Worker/Handlers/ScriptHandler.cs similarity index 51% rename from src/Agent.Worker/Handlers/ShellHandler.cs rename to src/Agent.Worker/Handlers/ScriptHandler.cs index 229160f8e56..1af132381bf 100644 --- a/src/Agent.Worker/Handlers/ShellHandler.cs +++ b/src/Agent.Worker/Handlers/ScriptHandler.cs @@ -1,110 +1,89 @@ -using Microsoft.VisualStudio.Services.Agent.Util; +using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; using System.Threading.Tasks; -using System; -using Newtonsoft.Json.Linq; -using System.Text.RegularExpressions; -using Microsoft.VisualStudio.Services.Agent.Worker.Container; -using System.Linq; +using Microsoft.VisualStudio.Services.Agent.Util; +using Build = Microsoft.TeamFoundation.Build.WebApi; +using DistributedTask = Microsoft.TeamFoundation.DistributedTask.WebApi; using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines; -using Microsoft.TeamFoundation.Build.WebApi; namespace Microsoft.VisualStudio.Services.Agent.Worker.Handlers { - [ServiceLocator(Default = typeof(ShellHandler))] - public interface IShellHandler : IHandler + [ServiceLocator(Default = typeof(ScriptHandler))] + public interface IScriptHandler : IHandler { - ShellHandlerData Data { get; set; } } - public sealed class ShellHandler : Handler, IShellHandler + public sealed class ScriptHandler : Handler, IScriptHandler { - public ShellHandlerData Data { get; set; } - public async Task RunAsync() { - // Validate args. + // Validate args Trace.Entering(); - ArgUtil.NotNull(Data, nameof(Data)); ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); ArgUtil.NotNull(Inputs, nameof(Inputs)); - // get the entry executable shell - // by default cmd on windows and bash on linux/mac - // user can overwrite this - string shell = Inputs.GetValueOrDefault("shell"); - if (string.IsNullOrEmpty(shell)) - { -#if OS_WINDOWS - // Resolve cmd.exe for windows - shell = "cmd.exe"; -#else - shell = "bash"; -#endif - } + var tempDirectory = HostContext.GetDirectory(WellKnownDirectory.Temp); - ArgUtil.NotNullOrEmpty(shell, nameof(shell)); - ExecutionContext.Output($"Use '{shell}' execute generated script."); + Inputs.TryGetValue("script", out var contents); + contents = contents ?? string.Empty; - // Write script to file - string script = Inputs.GetValueOrDefault("script"); - string scriptFileExtension = null; - if (shell.EndsWith("powershell", StringComparison.OrdinalIgnoreCase) || - shell.EndsWith("powershell.exe", StringComparison.OrdinalIgnoreCase)) + Inputs.TryGetValue("workingDirectory", out var workingDirectory); + if (string.IsNullOrEmpty(workingDirectory)) { - scriptFileExtension = "ps1"; + workingDirectory = ExecutionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory); + if (string.IsNullOrEmpty(workingDirectory)) + { + workingDirectory = HostContext.GetDirectory(WellKnownDirectory.Work); + } } - else - { + #if OS_WINDOWS - scriptFileExtension = "cmd"; - script = "@echo off" + System.Environment.NewLine + script; + // Fixup contents + contents = contents.Replace("\r\n", "\n").Replace("\n", "\r\n"); + // Note, use @echo off instead of using the /Q command line switch. + // When /Q is used, echo can't be turned on. + contents = $"@echo off\r\n{contents}"; + + // Write the script + var filePath = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.cmd"); + var encoding = ExecutionContext.Variables.Retain_Default_Encoding && Console.InputEncoding.CodePage != 65001 + ? Console.InputEncoding + : new UTF8Encoding(false); + File.WriteAllText(filePath, contents, encoding); + + // Command path + var commandPath = System.Environment.GetEnvironmentVariable("ComSpec"); + ArgUtil.NotNullOrEmpty(commandPath, "%ComSpec%"); + + // Arguments + var arguments = $"/D /E:ON /V:OFF /S /C \"CALL \"{StepHost.ResolvePathForStepHost(filePath)}\"\""; #else - scriptFileExtension = "sh"; - script = "set -eo pipefail" + System.Environment.NewLine + script; -#endif - } + // Fixup contents + contents = $"set -eo pipefail\n{contents}"; - ExecutionContext.Output("Script contents:"); - ExecutionContext.Output(script); - - string scriptFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Temp), $"{Guid.NewGuid()}.{scriptFileExtension}"); + // Write the script + var filePath = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.sh"); + // Don't add a BOM. It causes the script to fail on some operating systems (e.g. on Ubuntu 14). + File.WriteAllText(filePath, contents, new UTF8Encoding(false)); - ExecutionContext.Output($"Generate script file: {scriptFile}"); - File.WriteAllText(scriptFile, script, new UTF8Encoding(false)); + // Command path + var commandPath = WhichUtil.Which("bash") ?? WhichUtil.Which("sh", true); - // get arguments - // we support running cmd/bash/sh/powershell for now - string arguments; - if (scriptFileExtension.Equals("ps1", StringComparison.OrdinalIgnoreCase)) - { - // powershell -NoLogo -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command ". '.Replace("'", "''"))'" - arguments = $"-NoLogo -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command \". '{StepHost.ResolvePathForStepHost(scriptFile).Replace("'", "''")}'\""; - } - else if (scriptFileExtension.Equals("cmd", StringComparison.OrdinalIgnoreCase)) - { - // cmd /D /E:ON /V:OFF /S /C "CALL """ - arguments = $"/D /E:ON /V:OFF /S /C \"CALL \"{StepHost.ResolvePathForStepHost(scriptFile)}\"\""; - } - else - { - // bash --noprofile --norc "" - arguments = $"--noprofile --norc \"{StepHost.ResolvePathForStepHost(scriptFile)}\""; - } + // Arguments + var arguments = $"--noprofile --norc {StepHost.ResolvePathForStepHost(filePath).Replace("\"", "\\\"")}"; +#endif - // get working directory - string workingDirectory = Inputs.GetValueOrDefault("workingDirectory"); - if (string.IsNullOrEmpty(workingDirectory)) - { - workingDirectory = ExecutionContext.Variables.Get(Constants.Variables.System.DefaultWorkingDirectory); - } + ExecutionContext.Output("Script contents:"); + ExecutionContext.Output(contents); + ExecutionContext.Output("========================== Starting Command Output ==========================="); - // get environment variable + // Prepend PATH AddPrependPathToEnvironment(); - // populate action environment variables. + // Populate action environment variables var selfRepo = ExecutionContext.Repositories.Single(x => string.Equals(x.Alias, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase) || string.Equals(x.Alias, Pipelines.PipelineConstants.DesignerRepo, StringComparison.OrdinalIgnoreCase)); @@ -140,37 +119,27 @@ public async Task RunAsync() Environment["GITHUB_WORKFLOW"] = ExecutionContext.Variables.Build_DefinitionName; // GITHUB_EVENT_NAME=push - Environment["GITHUB_EVENT_NAME"] = ExecutionContext.Variables.Get(BuildVariables.Reason); + Environment["GITHUB_EVENT_NAME"] = ExecutionContext.Variables.Get(Build::BuildVariables.Reason); // GITHUB_ACTION=dump.env // GITHUB_EVENT_PATH=/github/workflow/event.json - // execute through stephost + // Execute through stephost StepHost.OutputDataReceived += OnDataReceived; StepHost.ErrorDataReceived += OnDataReceived; -#if OS_WINDOWS - // It appears that node.exe outputs UTF8 when not in TTY mode. - Encoding outputEncoding = Encoding.UTF8; -#else - // Let .NET choose the default. - Encoding outputEncoding = null; -#endif - - // Execute the process. Exit code 0 should always be returned. - // A non-zero exit code indicates infrastructural failure. - // Task failure should be communicated over STDOUT using ## commands. + // Execute int exitCode = await StepHost.ExecuteAsync(workingDirectory: StepHost.ResolvePathForStepHost(workingDirectory), - fileName: StepHost.ResolvePathForStepHost(shell), + fileName: StepHost.ResolvePathForStepHost(commandPath), arguments: arguments, environment: Environment, requireExitCodeZero: false, - outputEncoding: outputEncoding, + outputEncoding: null, killProcessOnCancel: false, inheritConsoleHandler: !ExecutionContext.Variables.Retain_Default_Encoding, cancellationToken: ExecutionContext.CancellationToken); - // Fail on non-zero exit code. + // Error if (exitCode != 0) { throw new Exception(StringUtil.Loc("ProcessCompletedWithExitCode0", exitCode)); diff --git a/src/Agent.Worker/JobExtension.cs b/src/Agent.Worker/JobExtension.cs index 51472687626..b2d2371803f 100644 --- a/src/Agent.Worker/JobExtension.cs +++ b/src/Agent.Worker/JobExtension.cs @@ -90,26 +90,29 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel Dictionary taskConditionMap = new Dictionary(); foreach (var step in message.Steps) { - if (step.Type == Pipelines.StepType.Task || - step.Type == Pipelines.StepType.Action) + IExpressionNode condition; + if (string.IsNullOrEmpty(step.Condition)) { - IExpressionNode condition; - if (!string.IsNullOrEmpty(step.Condition)) + condition = ExpressionManager.Succeeded; + } + else + { + context.Debug($"Step '{step.DisplayName}' has following condition: '{step.Condition}'."); + if (step.Type == Pipelines.StepType.Task) { - context.Debug($"Task '{step.DisplayName}' has following condition: '{step.Condition}'."); - condition = expression.Parse(context, step.Condition); + condition = expression.Parse(context, step.Condition, legacy: true); + } + else if (step.Type == Pipelines.StepType.Action || step.Type == Pipelines.StepType.Script) + { + condition = expression.Parse(context, step.Condition, legacy: false); } else { - condition = ExpressionManager.Succeeded; + throw new NotSupportedException(step.Type.ToString()); } - - taskConditionMap[step.Id] = condition; - } - else - { - throw new NotSupportedException(step.Type.ToString()); } + + taskConditionMap[step.Id] = condition; } #if OS_WINDOWS @@ -167,6 +170,15 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel actionRunner.Condition = taskConditionMap[action.Id]; jobSteps.Add(actionRunner); } + else if (step.Type == Pipelines.StepType.Script) + { + var script = step as Pipelines.ScriptStep; + Trace.Info($"Adding {script.DisplayName}."); + var scriptRunner = HostContext.CreateService(); + scriptRunner.Script = script; + scriptRunner.Condition = taskConditionMap[script.Id]; + jobSteps.Add(scriptRunner); + } else if (step.Type == Pipelines.StepType.Task) { var task = step as Pipelines.TaskStep; @@ -265,6 +277,12 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel ArgUtil.NotNull(actionStep, step.DisplayName); actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, outputForward: true); } + else if (step is IScriptRunner) + { + IScriptRunner scriptStep = step as IScriptRunner; + ArgUtil.NotNull(scriptStep, step.DisplayName); + scriptStep.ExecutionContext = jobContext.CreateChild(scriptStep.Script.Id, scriptStep.DisplayName, scriptStep.Script.Name, outputForward: true); + } } // Add post-job steps from Tasks diff --git a/src/Agent.Worker/ScriptRunner.cs b/src/Agent.Worker/ScriptRunner.cs new file mode 100644 index 00000000000..2f13ed9f361 --- /dev/null +++ b/src/Agent.Worker/ScriptRunner.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.TeamFoundation.DistributedTask.Expressions; +using Microsoft.TeamFoundation.DistributedTask.ObjectTemplating; +using Microsoft.TeamFoundation.DistributedTask.Pipelines.ObjectTemplating; +using Microsoft.TeamFoundation.DistributedTask.WebApi; +using Microsoft.VisualStudio.Services.Agent.Util; +using Microsoft.VisualStudio.Services.Agent.Worker.Container; +using Microsoft.VisualStudio.Services.Agent.Worker.Handlers; +using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines; + +namespace Microsoft.VisualStudio.Services.Agent.Worker +{ + [ServiceLocator(Default = typeof(ScriptRunner))] + public interface IScriptRunner : IStep, IAgentService + { + Pipelines.ScriptStep Script { get; set; } + } + + public sealed class ScriptRunner : AgentService, IScriptRunner + { + public IExpressionNode Condition { get; set; } + + public bool ContinueOnError => Script?.ContinueOnError ?? default(bool); + + public string DisplayName => Script?.DisplayName; + + public bool Enabled => Script?.Enabled ?? default(bool); + + public IExecutionContext ExecutionContext { get; set; } + + public Pipelines.ScriptStep Script { get; set; } + + public TimeSpan? Timeout => (Script?.TimeoutInMinutes ?? 0) > 0 ? (TimeSpan?)TimeSpan.FromMinutes(Script.TimeoutInMinutes) : null; + + public async Task RunAsync() + { + // Validate args. + Trace.Entering(); + ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext)); + ArgUtil.NotNull(Script, nameof(Script)); + var taskManager = HostContext.GetService(); + var handlerFactory = HostContext.GetService(); + + IStepHost stepHost = HostContext.CreateService(); + + // Setup container stephost for running inside the container. + if (ExecutionContext.Container != null) + { + // Make sure required container is already created. + ArgUtil.NotNullOrEmpty(ExecutionContext.Container.ContainerId, nameof(ExecutionContext.Container.ContainerId)); + var containerStepHost = HostContext.CreateService(); + containerStepHost.Container = ExecutionContext.Container; + stepHost = containerStepHost; + } + + // Load the inputs. + ExecutionContext.Output($"{WellKnownTags.Debug}Loading inputs"); + var templateTrace = ExecutionContext.ToTemplateTraceWriter(); + var schema = new PipelineTemplateSchemaFactory().CreateSchema(); + var templateEvaluator = new PipelineTemplateEvaluator(templateTrace, schema); + var inputs = templateEvaluator.EvaluateStepInputs(Script.Inputs, ExecutionContext.ExpressionValues); + + // Load the task environment. + ExecutionContext.Output($"{WellKnownTags.Debug}Loading env"); + var environment = templateEvaluator.EvaluateStepEnvironment(Script.Environment, ExecutionContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer); + + // Create the handler. + var handler = HostContext.CreateService(); + handler.ExecutionContext = ExecutionContext; + handler.StepHost = stepHost; + handler.Inputs = inputs; + handler.Environment = environment; + handler.RuntimeVariables = ExecutionContext.Variables; + + // Run the task. + await handler.RunAsync(); + } + } +} diff --git a/src/Agent.Worker/TaskManager.cs b/src/Agent.Worker/TaskManager.cs index 181ce7ff1da..dddcc1c3d5c 100644 --- a/src/Agent.Worker/TaskManager.cs +++ b/src/Agent.Worker/TaskManager.cs @@ -89,12 +89,6 @@ into taskGrouping continue; } - if (task.Id == Pipelines.PipelineConstants.RunTask.Id && task.Version == Pipelines.PipelineConstants.RunTask.Version) - { - Trace.Info("Skip download run task."); - continue; - } - await DownloadAsync(executionContext, task); } @@ -596,25 +590,6 @@ public Definition Load(Pipelines.TaskStep task) return checkoutTask; } - if (task.Reference.Id == Pipelines.PipelineConstants.RunTask.Id && task.Reference.Version == Pipelines.PipelineConstants.RunTask.Version) - { - var runTask = new Definition() - { - Directory = HostContext.GetDirectory(WellKnownDirectory.Tasks), - Data = new DefinitionData() - { - Author = Pipelines.PipelineConstants.RunTask.Author, - Description = Pipelines.PipelineConstants.RunTask.Description, - FriendlyName = Pipelines.PipelineConstants.RunTask.FriendlyName, - HelpMarkDown = Pipelines.PipelineConstants.RunTask.HelpMarkDown, - Inputs = Pipelines.PipelineConstants.RunTask.Inputs.ToArray(), - Execution = StringUtil.ConvertFromJson(StringUtil.ConvertToJson(Pipelines.PipelineConstants.RunTask.Execution)), - } - }; - - return runTask; - } - // if (task.Reference.Id == new Guid("22f9b24a-0e55-484c-870e-1a0041f0167e")) // { // var containerTask = new Definition() @@ -913,7 +888,6 @@ public sealed class ExecutionData private AzurePowerShellHandlerData _azurePowerShell; private ContainerActionHandlerData _containerAction; private NodeScriptActionHandlerData _nodeScriptAction; - private ShellHandlerData _shell; private NodeHandlerData _node; private Node10HandlerData _node10; private PowerShellHandlerData _powerShell; @@ -998,20 +972,6 @@ public NodeScriptActionHandlerData NodeAction } } - public ShellHandlerData Shell - { - get - { - return _shell; - } - - set - { - _shell = value; - Add(value); - } - } - #if !OS_WINDOWS || X86 [JsonIgnore] #endif @@ -1427,9 +1387,4 @@ public sealed class AgentPluginHandlerData : HandlerData { public override int Priority => 0; } - - public sealed class ShellHandlerData : HandlerData - { - public override int Priority => 6; - } } diff --git a/src/Agent.Worker/WorkerUtilties.cs b/src/Agent.Worker/WorkerUtilties.cs index 7e4c2254ce0..f0f82eb2cf6 100644 --- a/src/Agent.Worker/WorkerUtilties.cs +++ b/src/Agent.Worker/WorkerUtilties.cs @@ -89,6 +89,7 @@ public static Pipelines.AgentJobRequestMessage ScrubPiiData(Pipelines.AgentJobRe variables: scrubbedVariables, maskHints: message.MaskHints, jobResources: scrubbedJobResources, + expressionValues: message.ExpressionValues, workspaceOptions: message.Workspace, steps: message.Steps); } diff --git a/src/Microsoft.VisualStudio.Services.Agent/Microsoft.VisualStudio.Services.Agent.csproj b/src/Microsoft.VisualStudio.Services.Agent/Microsoft.VisualStudio.Services.Agent.csproj index e5ef55849bb..0640bc2082e 100644 --- a/src/Microsoft.VisualStudio.Services.Agent/Microsoft.VisualStudio.Services.Agent.csproj +++ b/src/Microsoft.VisualStudio.Services.Agent/Microsoft.VisualStudio.Services.Agent.csproj @@ -19,7 +19,7 @@ - + diff --git a/src/Test/Test.csproj b/src/Test/Test.csproj index 7fbb2b7b5ab..9e8a0ae23e0 100644 --- a/src/Test/Test.csproj +++ b/src/Test/Test.csproj @@ -22,7 +22,7 @@ - +