diff --git a/service/application/src/main/java/com/solace/maas/ep/common/messages/CommandLogMessage.java b/service/application/src/main/java/com/solace/maas/ep/common/messages/CommandLogMessage.java index cc8a30f0..1acf8d62 100644 --- a/service/application/src/main/java/com/solace/maas/ep/common/messages/CommandLogMessage.java +++ b/service/application/src/main/java/com/solace/maas/ep/common/messages/CommandLogMessage.java @@ -12,13 +12,16 @@ public class CommandLogMessage extends MOPMessage { String commandCorrelationId; + String eventManagementAgentId; + String level; String log; Long timestamp; - public CommandLogMessage(String orgId, String commandCorrelationId, String traceId, String actorId, String level, String log, Long timestamp) { + public CommandLogMessage(String orgId, String commandCorrelationId, String traceId, String actorId, + String level, String log, Long timestamp, String runtimeAgentId) { super(); withMessageType(MOPMessageType.generic) .withProtocol(MOPProtocol.epConfigPush) @@ -27,6 +30,7 @@ public CommandLogMessage(String orgId, String commandCorrelationId, String trace this.orgId = orgId; this.commandCorrelationId = commandCorrelationId; + eventManagementAgentId = runtimeAgentId; this.level = level; this.log = log; this.timestamp = timestamp; diff --git a/service/application/src/main/java/com/solace/maas/ep/event/management/agent/logging/CommandFilter.java b/service/application/src/main/java/com/solace/maas/ep/event/management/agent/logging/CommandFilter.java new file mode 100644 index 00000000..b4459cbb --- /dev/null +++ b/service/application/src/main/java/com/solace/maas/ep/event/management/agent/logging/CommandFilter.java @@ -0,0 +1,24 @@ +package com.solace.maas.ep.event.management.agent.logging; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.filter.Filter; +import ch.qos.logback.core.spi.FilterReply; +import com.solace.maas.ep.event.management.agent.plugin.constants.RouteConstants; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class CommandFilter extends Filter { + + @Override + public FilterReply decide(ILoggingEvent event) { + + String commandCorrelationId = event.getMDCPropertyMap().get(RouteConstants.COMMAND_CORRELATION_ID); + + if (StringUtils.isNotEmpty(commandCorrelationId)) { + return FilterReply.ACCEPT; + } + + return FilterReply.DENY; + } +} diff --git a/service/application/src/main/java/com/solace/maas/ep/event/management/agent/processor/CommandLogsProcessor.java b/service/application/src/main/java/com/solace/maas/ep/event/management/agent/processor/CommandLogsProcessor.java index 0e45b85e..348a9301 100644 --- a/service/application/src/main/java/com/solace/maas/ep/event/management/agent/processor/CommandLogsProcessor.java +++ b/service/application/src/main/java/com/solace/maas/ep/event/management/agent/processor/CommandLogsProcessor.java @@ -47,7 +47,7 @@ public void process(Exchange exchange) throws Exception { String messagingServiceId = (String) properties.get(RouteConstants.MESSAGING_SERVICE_ID); CommandLogMessage logDataMessage = new CommandLogMessage(orgId, commandCorrelationId, traceId, actorId, event.getLevel().toString(), - String.format("%s%s", event.getFormattedMessage(), "\n"), event.getTimeStamp()); + String.format("%s%s", event.getFormattedMessage(), "\n"), event.getTimeStamp(), runtimeAgentId); topicDetails.put("orgId", orgId); topicDetails.put("runtimeAgentId", runtimeAgentId); diff --git a/service/application/src/main/resources/command-configs.properties b/service/application/src/main/resources/command-configs.properties new file mode 100644 index 00000000..3310aa8a --- /dev/null +++ b/service/application/src/main/resources/command-configs.properties @@ -0,0 +1 @@ +COMMAND_PATH=${HOME}/commands \ No newline at end of file diff --git a/service/application/src/main/resources/logback-spring.xml b/service/application/src/main/resources/logback-spring.xml index 0f3f960e..1829d790 100644 --- a/service/application/src/main/resources/logback-spring.xml +++ b/service/application/src/main/resources/logback-spring.xml @@ -2,6 +2,7 @@ + @@ -35,17 +36,39 @@ + + + + + + ${COMMAND_PATH}/logs/command-logs.log + + + [%X{COMMAND_CORRELATION_ID}] ------ %date [%level] [%thread] %logger{10} [%file:%line] %msg%n + + + + ${COMMAND_PATH}/logs/command-logs-%d{yyyy-MM-dd}.%i.log + 1MB + 30 + 10MB + false + + + + + diff --git a/service/application/src/test/java/com/solace/maas/ep/event/management/agent/plugin/route/handler/CommandLogsPublisherRouteBuilderTests.java b/service/application/src/test/java/com/solace/maas/ep/event/management/agent/plugin/route/handler/CommandLogsPublisherRouteBuilderTests.java index 0e0be3b0..81dfb5f7 100644 --- a/service/application/src/test/java/com/solace/maas/ep/event/management/agent/plugin/route/handler/CommandLogsPublisherRouteBuilderTests.java +++ b/service/application/src/test/java/com/solace/maas/ep/event/management/agent/plugin/route/handler/CommandLogsPublisherRouteBuilderTests.java @@ -91,7 +91,8 @@ void testCommandLogMessageMOPProtocol() { "traceId", "actorId", "level", - "log", Instant.now().toEpochMilli()); + "log", Instant.now().toEpochMilli(), + "runtimeAgentId"); assertThat(commandLogMessage.getMopProtocol()).isEqualTo(MOPProtocol.epConfigPush); } diff --git a/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/configuration/TerraformProperties.java b/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/configuration/TerraformProperties.java index d12af5fe..87f1c588 100644 --- a/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/configuration/TerraformProperties.java +++ b/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/configuration/TerraformProperties.java @@ -7,6 +7,6 @@ @Service @Data public class TerraformProperties { - @Value("${plugins.terraform.workingDirectoryRoot:${HOME}/tfconfig}") + @Value("${COMMAND_PATH:${HOME}/tfconfig}") private String workingDirectoryRoot; } diff --git a/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/configuration/TerraformPropertiesFromFile.java b/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/configuration/TerraformPropertiesFromFile.java new file mode 100644 index 00000000..8b63f11d --- /dev/null +++ b/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/configuration/TerraformPropertiesFromFile.java @@ -0,0 +1,9 @@ +package com.solace.maas.ep.event.management.agent.plugin.terraform.configuration; + +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; + +@Configuration +@PropertySource("classpath:command-configs.properties") +public class TerraformPropertiesFromFile { +} diff --git a/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/manager/TerraformLogProcessingService.java b/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/manager/TerraformLogProcessingService.java index 50885bad..98b3d5b7 100644 --- a/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/manager/TerraformLogProcessingService.java +++ b/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/manager/TerraformLogProcessingService.java @@ -1,22 +1,14 @@ package com.solace.maas.ep.event.management.agent.plugin.terraform.manager; import com.fasterxml.jackson.databind.ObjectMapper; -import com.solace.maas.ep.event.management.agent.plugin.command.model.CommandRequest; import com.solace.maas.ep.event.management.agent.plugin.command.model.CommandResult; import com.solace.maas.ep.event.management.agent.plugin.command.model.JobStatus; -import com.solace.maas.ep.event.management.agent.plugin.terraform.configuration.TerraformProperties; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang.StringUtils; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.List; import java.util.Map; @@ -33,32 +25,10 @@ public class TerraformLogProcessingService { public static final String KEY_TIMESTAMP = "@timestamp"; public static final String KEY_DIAGNOSTIC_DETAIL = "diagnosticDetail"; public static final String KEY_DIAGNOSTIC = "diagnostic"; - private final TerraformProperties terraformProperties; private final ObjectMapper objectMapper; - public TerraformLogProcessingService(ObjectMapper objectMapper, TerraformProperties terraformProperties) { + public TerraformLogProcessingService(ObjectMapper objectMapper) { this.objectMapper = objectMapper; - this.terraformProperties = terraformProperties; - } - - public void saveLogToFile(CommandRequest request, List logs) throws IOException { - Path logPath = Paths.get(terraformProperties.getWorkingDirectoryRoot() - + File.separator - + request.getContext() - + "-" - + request.getServiceId() - + File.separator - + "logs" - ); - - if (Files.notExists(logPath)) { - Files.createDirectories(logPath); - } - - Path out = Files.createTempFile(logPath, System.currentTimeMillis() + "-" - + request.getCommandCorrelationId() + "-job", ".log"); - - Files.write(out, logs, Charset.defaultCharset()); } public CommandResult buildTfCommandResult(List jsonLogs) { diff --git a/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/manager/TerraformManager.java b/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/manager/TerraformManager.java index 93d35c31..d92e76b4 100644 --- a/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/manager/TerraformManager.java +++ b/service/terraform-plugin/src/main/java/com/solace/maas/ep/event/management/agent/plugin/terraform/manager/TerraformManager.java @@ -1,5 +1,6 @@ package com.solace.maas.ep.event.management.agent.plugin.terraform.manager; +import com.fasterxml.jackson.databind.ObjectMapper; import com.solace.maas.ep.event.management.agent.plugin.command.model.Command; import com.solace.maas.ep.event.management.agent.plugin.command.model.CommandRequest; import com.solace.maas.ep.event.management.agent.plugin.command.model.CommandResult; @@ -8,6 +9,7 @@ import com.solace.maas.ep.event.management.agent.plugin.terraform.client.TerraformClient; import com.solace.maas.ep.event.management.agent.plugin.terraform.client.TerraformClientFactory; import com.solace.maas.ep.event.management.agent.plugin.terraform.configuration.TerraformProperties; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang.StringUtils; import org.slf4j.MDC; @@ -29,10 +31,11 @@ @Slf4j public class TerraformManager { public static final String LOG_LEVEL_ERROR = "ERROR"; + private static final ObjectMapper objectMapper = new ObjectMapper(); private final TerraformLogProcessingService terraformLogProcessingService; private final TerraformProperties terraformProperties; private final TerraformClientFactory terraformClientFactory; - private final static String TF_CONFIG_FILENAME = "config.tf"; + private static final String TF_CONFIG_FILENAME = "config.tf"; public TerraformManager(TerraformLogProcessingService terraformLogProcessingService, TerraformProperties terraformProperties, TerraformClientFactory terraformClientFactory) { @@ -54,7 +57,7 @@ public void execute(CommandRequest request, Command command, Map Path configPath = createConfigPath(request); List logOutput = setupTerraformClient(terraformClient, configPath); String commandVerb = executeTerraformCommand(command, envVars, configPath, terraformClient); - processTerraformResponse(request, command, commandVerb, logOutput); + processTerraformResponse(command, commandVerb, logOutput); } catch (InterruptedException e) { log.error("Received a thread interrupt while executing the terraform command", e); Thread.currentThread().interrupt(); @@ -72,11 +75,28 @@ private static List setupTerraformClient(TerraformClient terraformClient // Also write the output to the main log to be streamed back to EP terraformClient.setOutputListener(tfLog -> { output.add(tfLog); - log.debug("Terraform output: {}", tfLog); + logToConsole(tfLog); }); return output; } + @SneakyThrows + private static void logToConsole(String tfLog) { + + String logMessage = String.format("Terraform output: %s", tfLog); + + Map logMop = objectMapper.readValue(tfLog, Map.class); + String logLevel = (String) logMop.get("@level"); + switch (logLevel) { + case "trace" -> log.trace(logMessage); + case "debug" -> log.debug(logMessage); + case "info" -> log.info(logMessage); + case "warn" -> log.warn(logMessage); + case "error" -> log.error(logMessage); + default -> log.error("cannot map the logLevel properly for tfLog {}", tfLog); + } + } + private static String executeTerraformCommand(Command command, Map envVars, Path configPath, TerraformClient terraformClient) throws IOException, InterruptedException, ExecutionException { String commandVerb = command.getCommand(); switch (commandVerb) { @@ -93,7 +113,7 @@ private static String executeTerraformCommand(Command command, Map output) throws IOException { + private void processTerraformResponse(Command command, String commandVerb, List output) { // Process logs and create the result if (Boolean.TRUE.equals(command.getIgnoreResult())) { command.setResult(CommandResult.builder() @@ -102,7 +122,6 @@ private void processTerraformResponse(CommandRequest request, Command command, S .build()); } else { if (!"write_HCL".equals(commandVerb)) { - terraformLogProcessingService.saveLogToFile(request, output); command.setResult(terraformLogProcessingService.buildTfCommandResult(output)); } else { command.setResult(CommandResult.builder() diff --git a/service/terraform-plugin/src/test/java/com/solace/maas/ep/event/management/agent/plugin/terraform/TerraformTestConfig.java b/service/terraform-plugin/src/test/java/com/solace/maas/ep/event/management/agent/plugin/terraform/TerraformTestConfig.java index 01057db6..117d2500 100644 --- a/service/terraform-plugin/src/test/java/com/solace/maas/ep/event/management/agent/plugin/terraform/TerraformTestConfig.java +++ b/service/terraform-plugin/src/test/java/com/solace/maas/ep/event/management/agent/plugin/terraform/TerraformTestConfig.java @@ -32,8 +32,8 @@ public TerraformProperties getTfProperties() { @Bean @Primary - public TerraformLogProcessingService getTfLogProcessingService(ObjectMapper objectMapper, TerraformProperties terraformProperties) { - return new TerraformLogProcessingService(objectMapper, terraformProperties); + public TerraformLogProcessingService getTfLogProcessingService(ObjectMapper objectMapper) { + return new TerraformLogProcessingService(objectMapper); } @Bean