Merge pull request #1222 from k2io/v2/nr/instrumentation-changes

Changes for NR CSEC module support

agent-bridge-datastore/build.gradle

@@ -1,3 +1,43 @@
+import com.nr.builder.publish.PublishConfig
+
+plugins {
+    id("maven-publish")
+    id("signing")
+}
+
+jar {
+    from("$rootDir/LICENSE")
+    manifest {
+        attributes 'Implementation-Title': 'New Relic Bridge DataStore', 'Implementation-Version': project.version
+    }
+}
+
+java {
+    withSourcesJar()
+    withJavadocJar()
+}
+
+tasks.withType(GenerateModuleMetadata.class) {
+    enabled = false
+}
+
+
+tasks.withType(Javadoc).configureEach {
+    javadocTool = javaToolchains.javadocToolFor {
+        languageVersion = JavaLanguageVersion.of(11)
+    }
+
+    options.addBooleanOption("-frames", true)
+    options.addBooleanOption("-no-module-directories", true)
+}
+
+PublishConfig.config(
+        project,
+        "New Relic Java Bridge",
+        "Bridge DataStore") { it ->
+    it.from(components.java)
+}
+
 dependencies {
     implementation(project(":newrelic-api"))
     implementation(project(":newrelic-weaver-api"))

agent-bridge/build.gradle

@@ -1,3 +1,43 @@
+import com.nr.builder.publish.PublishConfig
+
+plugins {
+    id("maven-publish")
+    id("signing")
+}
+
+jar {
+    from("$rootDir/LICENSE")
+    manifest {
+        attributes 'Implementation-Title': 'New Relic Bridge', 'Implementation-Version': project.version
+    }
+}
+
+java {
+    withSourcesJar()
+    withJavadocJar()
+}
+
+tasks.withType(GenerateModuleMetadata.class) {
+    enabled = false
+}
+
+
+tasks.withType(Javadoc).configureEach {
+    javadocTool = javaToolchains.javadocToolFor {
+        languageVersion = JavaLanguageVersion.of(11)
+    }
+
+    options.addBooleanOption("-frames", true)
+    options.addBooleanOption("-no-module-directories", true)
+}
+
+PublishConfig.config(
+        project,
+        "New Relic Java Bridge",
+        "Bridge") { it ->
+    it.from(components.java)
+}
+
 dependencies {
     api(project(":newrelic-api"))
     api(project(":newrelic-weaver-api"))

agent-bridge/src/main/java/com/newrelic/agent/bridge/NoOpTransaction.java

@@ -7,16 +7,9 @@
 
 package com.newrelic.agent.bridge;
 
-import com.newrelic.api.agent.ApplicationNamePriority;
+import com.newrelic.api.agent.*;
 import com.newrelic.api.agent.DistributedTracePayload;
-import com.newrelic.api.agent.ExtendedRequest;
-import com.newrelic.api.agent.Headers;
-import com.newrelic.api.agent.InboundHeaders;
-import com.newrelic.api.agent.Request;
-import com.newrelic.api.agent.Response;
-import com.newrelic.api.agent.Segment;
 import com.newrelic.api.agent.TransactionNamePriority;
-import com.newrelic.api.agent.TransportType;
 
 import java.net.URI;
 import java.util.Collection;
@@ -318,6 +311,11 @@ public void insertDistributedTraceHeaders(Headers headers) {
     public void acceptDistributedTraceHeaders(TransportType transportType, Headers headers) {
     }
 
+    @Override
+    public Object getSecurityMetaData() {
+        return new Object();
+    }
+
     @Override
     public void setTransportType(TransportType transportType) {
     }

build.gradle

@@ -19,6 +19,7 @@ allprojects {
 
     group = 'com.newrelic.agent.java'
     version = agentVersion + (project.findProperty("release") == "true" ? "" : "-SNAPSHOT")
+    version = version + (project.findProperty("release-suffix") != null ? project.findProperty("release-suffix") : "")
 
     idea.module {
         outputDir file('build/classes/main')

gradle.properties

@@ -1,5 +1,6 @@
 # The agent version.
 agentVersion=8.2.0
+csecCollectorVersion=1.0.1-SNAPSHOT
 
 newrelicDebug=false
 org.gradle.jvmargs=-Xmx2048m

newrelic-agent/build.gradle

@@ -14,6 +14,7 @@ plugins {
 
 repositories {
     mavenCentral()
+    mavenLocal()
     jcenter()
 }
 
@@ -59,6 +60,8 @@ dependencies {
     jarIntoJar project(":agent-bridge-datastore")
     jarIntoJar project(":newrelic-weaver-api")
     jarIntoJar project(":newrelic-weaver-scala-api")
+    jarIntoJar("com.newrelic.agent.java.security:newrelic-security-api:${csecCollectorVersion}")
+    jarIntoJar("com.newrelic.agent.java.security:newrelic-security-agent:${csecCollectorVersion}")
 
     shadowIntoJar project(":agent-interfaces")
     shadowIntoJar project(":agent-model")
@@ -263,6 +266,7 @@ task newrelicVersionedAgentJar(type: Jar) {
     // All projects have the same version, so we can safely remove this version from the name.
     from(project.configurations.jarIntoJar) {
         rename("-${project.version}", "")
+        rename("-${csecCollectorVersion}", "")
     }
 
     from(zipTree(project.tasks['transformedShadowJar'].archiveFile.get().asFile.path))

newrelic-agent/src/main/java/com/newrelic/agent/Agent.java

@@ -9,11 +9,7 @@
 
 import com.google.common.collect.ImmutableMap;
 import com.newrelic.agent.bridge.AgentBridge;
-import com.newrelic.agent.config.AgentJarHelper;
-import com.newrelic.agent.config.ConfigService;
-import com.newrelic.agent.config.ConfigServiceFactory;
-import com.newrelic.agent.config.JarResource;
-import com.newrelic.agent.config.JavaVersionUtils;
+import com.newrelic.agent.config.*;
 import com.newrelic.agent.core.CoreService;
 import com.newrelic.agent.core.CoreServiceImpl;
 import com.newrelic.agent.logging.AgentLogManager;
@@ -27,7 +23,10 @@
 import com.newrelic.agent.util.UnwindableInstrumentationImpl;
 import com.newrelic.agent.util.asm.ClassStructure;
 import com.newrelic.bootstrap.BootstrapAgent;
+import com.newrelic.api.agent.NewRelic;
+import com.newrelic.api.agent.security.NewRelicSecurity;
 import com.newrelic.bootstrap.BootstrapLoader;
+import com.newrelic.bootstrap.EmbeddedJarFilesImpl;
 import com.newrelic.weave.utils.Streams;
 import org.objectweb.asm.ClassReader;
 
@@ -227,6 +226,41 @@ public static void continuePremain(String agentArgs, Instrumentation inst, long
             instrumentation.started();
         }
         lifecycleObserver.agentStarted();
+        InitialiseNewRelicSecurityIfAllowed(inst);
+    }
+
+    private static void InitialiseNewRelicSecurityIfAllowed(Instrumentation inst) {
+        // Do not initialise New Relic Security module so that it stays in NoOp mode if force disabled.
+        if(NewRelic.getAgent().getConfig().getValue("security.agent.enabled", true) &&
+                NewRelic.getAgent().getConfig().getValue("security.enabled") != null) {
+            try {
+                LOG.log(Level.INFO, "Invoking New Relic Security module");
+                ServiceFactory.getServiceManager().getRPMServiceManager().addConnectionListener(new ConnectionListener() {
+                    @Override
+                    public void connected(IRPMService rpmService, AgentConfig agentConfig) {
+                        try {
+                            URL securityJarURL = EmbeddedJarFilesImpl.INSTANCE.getJarFileInAgent(BootstrapLoader.NEWRELIC_SECURITY_AGENT).toURI().toURL();
+                            LOG.log(Level.INFO, "Connected to New Relic cloud. Starting New Relic Security module");
+                            NewRelicSecurity.getAgent().refreshState(securityJarURL, inst);
+                            NewRelicSecurity.markAgentAsInitialised();
+                        } catch (IOException e) {
+                            throw new RuntimeException(e);
+                        }
+                    }
+
+                    @Override
+                    public void disconnected(IRPMService rpmService) {
+                        NewRelicSecurity.getAgent().deactivateSecurity();
+                    }
+                });
+            } catch (Throwable t2) {
+                // TODO : Give a generic error here.
+                LOG.error("license_key is empty in the config. Not starting New Relic Security Agent.");
+            }
+        } else {
+            LOG.warning("New Relic Security is completely disabled forcefully by user provided config `security.agent.enabled`. " +
+                    "Not loading security capabilities.");
+        }
     }
 
     private static Instrumentation maybeWrapInstrumentation(Instrumentation inst) {
@@ -255,6 +289,7 @@ private static boolean tryToInitializeServiceManager(Instrumentation inst) {
 
             // Now that we know the agent is enabled, add the ApiClassTransformer
             BootstrapLoader.forceCorrectNewRelicApi(inst);
+            BootstrapLoader.forceCorrectNewRelicSecurityApi(inst);
 
             // init problem classes before class transformer service is active
             InitProblemClasses.loadInitialClasses();

newrelic-agent/src/main/java/com/newrelic/agent/Transaction.java

@@ -83,6 +83,7 @@
 import com.newrelic.api.agent.NewRelic;
 import com.newrelic.api.agent.Request;
 import com.newrelic.api.agent.Response;
+import com.newrelic.api.agent.security.schema.SecurityMetaData;
 import org.objectweb.asm.Opcodes;
 
 import java.lang.management.GarbageCollectorMXBean;
@@ -242,6 +243,8 @@ public AppNameAndConfig call() throws Exception {
     // count of active tokens and tracers
     private final AtomicInteger activeCount;
 
+    private final SecurityMetaData securityMetaData;
+
     private final MetricAggregator metricAggregator = new AbstractMetricAggregator() {
         @Override
         protected void doRecordResponseTimeMetric(String name, long totalTime, long exclusiveTime, TimeUnit timeUnit) {
@@ -456,6 +459,7 @@ protected Transaction() {
         runningChildren = new LazyMapImpl<>(factory);
         activeTokensCache = new AtomicReference<>();
         activeCount = new AtomicInteger(0);
+        securityMetaData = new SecurityMetaData();
     }
 
     // This method must be called after construction. This is ugly, but avoids
@@ -2562,4 +2566,8 @@ int getCountOfRunningAndFinishedTransactionActivities() {
             return runningChildren.size() + finishedChildren.size();
         }
     }
+
+    public SecurityMetaData getSecurityMetaData() {
+        return securityMetaData;
+    }
 }

newrelic-agent/src/main/java/com/newrelic/agent/TransactionApiImpl.java

@@ -17,21 +17,11 @@
 import com.newrelic.agent.bridge.NoOpWebResponse;
 import com.newrelic.agent.bridge.Token;
 import com.newrelic.agent.bridge.TracedActivity;
-import com.newrelic.api.agent.Headers;
-import com.newrelic.api.agent.NewRelic;
-import com.newrelic.api.agent.TransportType;
+import com.newrelic.api.agent.*;
 import com.newrelic.agent.bridge.WebResponse;
 import com.newrelic.agent.service.ServiceFactory;
 import com.newrelic.agent.tracers.Tracer;
-import com.newrelic.api.agent.ApplicationNamePriority;
-import com.newrelic.api.agent.DistributedTracePayload;
-import com.newrelic.api.agent.ExtendedRequest;
-import com.newrelic.api.agent.InboundHeaders;
-import com.newrelic.api.agent.Request;
-import com.newrelic.api.agent.Response;
 import com.newrelic.api.agent.Segment;
-import com.newrelic.api.agent.TracedMethod;
-import com.newrelic.api.agent.TransactionNamePriority;
 
 import java.net.URI;
 import java.util.Map;
@@ -523,6 +513,15 @@ public void acceptDistributedTraceHeaders(TransportType transportType, Headers h
         HeadersUtil.parseAndAcceptDistributedTraceHeaders(tx, headers);
     }
 
+    @Override
+    public Object getSecurityMetaData() {
+        Transaction tx = getTransactionIfExists();
+        if (tx != null) {
+            return tx.getSecurityMetaData();
+        }
+        return null;
+    }
+
     @Override
     public void setTransportType(TransportType transportType) {
         Transaction tx = getTransactionIfExists();

newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigFactory.java

@@ -13,6 +13,7 @@
 import com.newrelic.agent.config.internal.DeepMapClone;
 import com.newrelic.agent.database.SqlObfuscator;
 import com.newrelic.agent.reinstrument.RemoteInstrumentationServiceImpl;
+import com.newrelic.agent.transport.ConnectionResponse;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -199,6 +200,13 @@ public static void mergeServerData(Map<String, Object> settings, Map<String, Obj
         addServerProp(EXPECTED_CLASSES, serverData.get(ErrorCollectorConfigImpl.EXPECTED_CLASSES), settings);
         addServerProp(EXPECTED_STATUS_CODES, serverData.get(ErrorCollectorConfigImpl.EXPECTED_STATUS_CODES), settings);
 
+        // Adding agent_run_id & account_id to config as required by K2 agent
+        addServerProp(ConnectionResponse.AGENT_RUN_ID_KEY, serverData.get(ConnectionResponse.AGENT_RUN_ID_KEY), settings);
+        addServerProp(DistributedTracingConfig.ACCOUNT_ID, serverData.get(DistributedTracingConfig.ACCOUNT_ID), settings);
+        addServerProp("agent_home", ConfigFileHelper.getNewRelicDirectory().getAbsolutePath(), settings);
+        if(AgentJarHelper.getAgentJarDirectory() != null) {
+            addServerProp("agent_jar_location", AgentJarHelper.getAgentJarDirectory().getAbsolutePath(), settings);
+        }
         if (settingsConfig.getProperty(SECURITY_POLICIES_TOKEN) != null) {
             addServerProp(RECORD_SQL, recordSqlSecure, settings);
             // Root

newrelic-agent/src/main/java/com/newrelic/agent/config/AgentConfigImpl.java

@@ -10,6 +10,7 @@
 import com.google.common.base.Joiner;
 import com.newrelic.agent.Agent;
 import com.newrelic.agent.DebugFlag;
+import com.newrelic.agent.security.intcodeagent.websocket.JsonConverter;
 import com.newrelic.agent.transaction.TransactionNamingScheme;
 import com.newrelic.agent.transport.DataSenderImpl;
 import com.newrelic.agent.util.Strings;
@@ -782,9 +783,38 @@ private BrowserMonitoringConfig initBrowserMonitoringConfig() {
     private ClassTransformerConfig initClassTransformerConfig(boolean liteMode) {
         boolean customTracingEnabled = getProperty(ENABLE_CUSTOM_TRACING, DEFAULT_ENABLE_CUSTOM_TRACING);
         Map<String, Object> props = nestedProps(CLASS_TRANSFORMER);
+        props = placeSecurityCollectorRelatedModification(props);
         return ClassTransformerConfigImpl.createClassTransformerConfig(props, customTracingEnabled, liteMode);
     }
 
+    /**
+     * CSEC specific excludes needed to allow functioning with java.io.InputStream and OutputStream instrumentation.
+     */
+    private Map<String, Object> placeSecurityCollectorRelatedModification(Map<String, Object> props) {
+        if(getProperty("security") != null) {
+            if(props == null) {
+                props = new HashMap<>();
+            } else {
+                props = new HashMap<>(props);
+            }
+            Set<String> securityExcludes = new HashSet<>() ;
+            securityExcludes.add("java/util/zip/InflaterInputStream");
+            securityExcludes.add("java/util/zip/ZipFile$ZipFileInputStream");
+            securityExcludes.add("java/util/zip/ZipFile$ZipFileInflaterInputStream");
+            securityExcludes.add("com/newrelic/.*");
+            securityExcludes.add("com/nr/.*");
+
+            Object userProvidedExcludes = props.get(ClassTransformerConfigImpl.EXCLUDES);
+            if(userProvidedExcludes instanceof String) {
+                securityExcludes.add((String) userProvidedExcludes);
+            } else if(userProvidedExcludes instanceof Set){
+                securityExcludes.addAll((Collection<? extends String>) userProvidedExcludes);
+            }
+            props.put(ClassTransformerConfigImpl.EXCLUDES, securityExcludes);
+        }
+        return props;
+    }
+
     private CircuitBreakerConfig initCircuitBreakerConfig() {
         Map<String, Object> props = nestedProps(CircuitBreakerConfig.PROPERTY_NAME);
         return new CircuitBreakerConfig(props);

newrelic-agent/src/main/java/com/newrelic/agent/config/AgentJarHelper.java

@@ -12,10 +12,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.net.URLDecoder;
+import java.net.*;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;