From 31e2e9c95d5a72f391df28c86adde926e5267d71 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 20 Jan 2022 10:37:30 +0100 Subject: [PATCH 1/8] draft --- x-pack/plugin/core/src/main/config/log4j2.properties | 1 + .../xpack/security/audit/logfile/LoggingAuditTrail.java | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index 3df7f338aff0b..9dddca2ad5113 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -37,6 +37,7 @@ appender.audit_rolling.layout.pattern = {\ %varsNotEmpty{, "request.name":"%enc{%map{request.name}}{JSON}"}\ %varsNotEmpty{, "indices":%map{indices}}\ %varsNotEmpty{, "opaque_id":"%enc{%map{opaque_id}}{JSON}"}\ + %varsNotEmpty{, "trace.id":"%enc{%map{trace_id}}{JSON}"}\ %varsNotEmpty{, "x_forwarded_for":"%enc{%map{x_forwarded_for}}{JSON}"}\ %varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}\ %varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}\ diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 1c453bf665d1a..9dc78d257c74a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -180,6 +180,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener { public static final String TRANSPORT_PROFILE_FIELD_NAME = "transport.profile"; public static final String RULE_FIELD_NAME = "rule"; public static final String OPAQUE_ID_FIELD_NAME = "opaque_id"; + public static final String TRACE_ID_FIELD_NAME = "trace_id"; public static final String X_FORWARDED_FOR_FIELD_NAME = "x_forwarded_for"; // the fields below are used exclusively for "security_config_change" type of events, and show the configuration // object taking effect; it could be creating a new, or updating an existing configuration @@ -1478,6 +1479,10 @@ LogEntryBuilder withOpaqueId(ThreadContext threadContext) { if (opaqueId != null) { logEntry.with(OPAQUE_ID_FIELD_NAME, opaqueId); } + final String traceId = threadContext.getHeader(Task.TRACE_ID); + if (traceId != null) { + logEntry.with(TRACE_ID_FIELD_NAME, opaqueId); + } return this; } From c261ef73b53ba5756e17ebeba96d9dc3fa7cdce3 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 20 Jan 2022 12:08:54 +0100 Subject: [PATCH 2/8] tests and renames --- .../core/src/main/config/log4j2.properties | 2 +- .../audit/logfile/LoggingAuditTrail.java | 83 ++++++++----------- .../audit/logfile/LoggingAuditTrailTests.java | 61 ++++++++++++-- 3 files changed, 89 insertions(+), 57 deletions(-) diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index 9dddca2ad5113..a4ed55d8ea778 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -37,7 +37,7 @@ appender.audit_rolling.layout.pattern = {\ %varsNotEmpty{, "request.name":"%enc{%map{request.name}}{JSON}"}\ %varsNotEmpty{, "indices":%map{indices}}\ %varsNotEmpty{, "opaque_id":"%enc{%map{opaque_id}}{JSON}"}\ - %varsNotEmpty{, "trace.id":"%enc{%map{trace_id}}{JSON}"}\ + %varsNotEmpty{, "trace.id":"%enc{%map{trace.id}}{JSON}"}\ %varsNotEmpty{, "x_forwarded_for":"%enc{%map{x_forwarded_for}}{JSON}"}\ %varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}\ %varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}\ diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 9dc78d257c74a..c2b35a55c08b5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -180,7 +180,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener { public static final String TRANSPORT_PROFILE_FIELD_NAME = "transport.profile"; public static final String RULE_FIELD_NAME = "rule"; public static final String OPAQUE_ID_FIELD_NAME = "opaque_id"; - public static final String TRACE_ID_FIELD_NAME = "trace_id"; + public static final String TRACE_ID_FIELD_NAME = "trace.id"; public static final String X_FORWARDED_FOR_FIELD_NAME = "x_forwarded_for"; // the fields below are used exclusively for "security_config_change" type of events, and show the configuration // object taking effect; it could be creating a new, or updating an existing configuration @@ -458,8 +458,7 @@ public void authenticationSuccess(String requestId, Authentication authenticatio .withAuthentication(authentication) .withRestOrigin(request) .withRequestBody(request) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -487,8 +486,7 @@ public void authenticationSuccess(String requestId, Authentication authenticatio .withAuthentication(authentication) .withRestOrTransportOrigin(transportRequest, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -507,8 +505,7 @@ public void anonymousAccessDenied(String requestId, String action, TransportRequ .withRequestId(requestId) .withRestOrTransportOrigin(transportRequest, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -524,8 +521,7 @@ public void anonymousAccessDenied(String requestId, RestRequest request) { .withRestOrigin(request) .withRequestBody(request) .withRequestId(requestId) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -544,8 +540,7 @@ public void authenticationFailed(String requestId, AuthenticationToken token, St .withRequestId(requestId) .withRestOrTransportOrigin(transportRequest, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext); + .withThreadContext(threadContext); if (token instanceof ServiceAccountToken) { logEntryBuilder.with(SERVICE_TOKEN_NAME_FIELD_NAME, ((ServiceAccountToken) token).getTokenName()); } @@ -563,8 +558,7 @@ public void authenticationFailed(String requestId, RestRequest request) { .withRestOrigin(request) .withRequestBody(request) .withRequestId(requestId) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -582,8 +576,7 @@ public void authenticationFailed(String requestId, String action, TransportReque .withRequestId(requestId) .withRestOrTransportOrigin(transportRequest, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -601,8 +594,7 @@ public void authenticationFailed(String requestId, AuthenticationToken token, Re .withRestOrigin(request) .withRequestBody(request) .withRequestId(requestId) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext); + .withThreadContext(threadContext); if (token instanceof ServiceAccountToken) { logEntryBuilder.with(SERVICE_TOKEN_NAME_FIELD_NAME, ((ServiceAccountToken) token).getTokenName()); } @@ -631,8 +623,7 @@ public void authenticationFailed( .withRequestId(requestId) .withRestOrTransportOrigin(transportRequest, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -651,8 +642,7 @@ public void authenticationFailed(String requestId, String realm, AuthenticationT .withRestOrigin(request) .withRequestBody(request) .withRequestId(requestId) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -688,8 +678,7 @@ public void accessGranted( .withAuthentication(authentication) .withRestOrTransportOrigin(msg, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .with(authorizationInfo.asMap()) .build(); } @@ -796,8 +785,7 @@ public void explicitIndexAccessEvent( .withRequestId(requestId) .withAuthentication(authentication) .with(INDICES_FIELD_NAME, indices) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .with(authorizationInfo.asMap()); final InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext); if (restAddress != null) { @@ -842,8 +830,7 @@ public void accessDenied( .withRestOrTransportOrigin(transportRequest, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) .with(authorizationInfo.asMap()) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -858,8 +845,7 @@ public void tamperedRequest(String requestId, RestRequest request) { .withRestOrigin(request) .withRequestBody(request) .withRequestId(requestId) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -877,8 +863,7 @@ public void tamperedRequest(String requestId, String action, TransportRequest tr .withRequestId(requestId) .withRestOrTransportOrigin(transportRequest, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -907,8 +892,7 @@ public void tamperedRequest(String requestId, Authentication authentication, Str .withRestOrTransportOrigin(transportRequest, threadContext) .withAuthentication(authentication) .with(INDICES_FIELD_NAME, indices.orElse(null)) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -926,8 +910,7 @@ public void connectionGranted(InetAddress inetAddress, String profile, SecurityI .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(inetAddress)) .with(TRANSPORT_PROFILE_FIELD_NAME, profile) .with(RULE_FIELD_NAME, rule.toString()) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -944,8 +927,7 @@ public void connectionDenied(InetAddress inetAddress, String profile, SecurityIp .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(inetAddress)) .with(TRANSPORT_PROFILE_FIELD_NAME, profile) .with(RULE_FIELD_NAME, rule.toString()) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -980,8 +962,7 @@ public void runAsGranted( .withRestOrTransportOrigin(transportRequest, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) .with(authorizationInfo.asMap()) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -1017,8 +998,7 @@ public void runAsDenied( .withRestOrTransportOrigin(transportRequest, threadContext) .with(INDICES_FIELD_NAME, indices.orElse(null)) .with(authorizationInfo.asMap()) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -1046,8 +1026,7 @@ public void runAsDenied(String requestId, Authentication authentication, RestReq .withRestOrigin(request) .withRequestBody(request) .withRequestId(requestId) - .withOpaqueId(threadContext) - .withXForwardedFor(threadContext) + .withThreadContext(threadContext) .build(); } } @@ -1474,7 +1453,7 @@ LogEntryBuilder withRequestId(String requestId) { return this; } - LogEntryBuilder withOpaqueId(ThreadContext threadContext) { + LogEntryBuilder withXOpaqueIdAndTraceId(ThreadContext threadContext) { final String opaqueId = threadContext.getHeader(Task.X_OPAQUE_ID_HTTP_HEADER); if (opaqueId != null) { logEntry.with(OPAQUE_ID_FIELD_NAME, opaqueId); @@ -1486,14 +1465,20 @@ LogEntryBuilder withOpaqueId(ThreadContext threadContext) { return this; } - LogEntryBuilder withXForwardedFor(ThreadContext threadContext) { - final String xForwardedFor = threadContext.getHeader(AuditTrail.X_FORWARDED_FOR_HEADER); - if (xForwardedFor != null) { - logEntry.with(X_FORWARDED_FOR_FIELD_NAME, xForwardedFor); - } + LogEntryBuilder withThreadContext(ThreadContext threadContext) { + setThreadContextField(threadContext, AuditTrail.X_FORWARDED_FOR_HEADER, X_FORWARDED_FOR_FIELD_NAME); + setThreadContextField(threadContext, Task.X_OPAQUE_ID_HTTP_HEADER, OPAQUE_ID_FIELD_NAME); + setThreadContextField(threadContext, Task.TRACE_ID, TRACE_ID_FIELD_NAME); return this; } + private void setThreadContextField(ThreadContext threadContext, String threadContextFieldName, String auditLogFieldName) { + final String fieldValue = threadContext.getHeader(threadContextFieldName); + if (fieldValue != null) { + logEntry.with(auditLogFieldName, fieldValue); + } + } + LogEntryBuilder withAuthentication(Authentication authentication) { logEntry.with(PRINCIPAL_FIELD_NAME, authentication.getUser().principal()); logEntry.with(AUTHENTICATION_TYPE_FIELD_NAME, authentication.getAuthenticationType().toString()); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index c64195c569571..d4f76093f56f2 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -325,6 +325,9 @@ public void init() throws Exception { if (randomBoolean()) { threadContext.putHeader(Task.X_OPAQUE_ID_HTTP_HEADER, randomAlphaOfLengthBetween(1, 4)); } + if (randomBoolean()) { + threadContext.putHeader(Task.TRACE_ID, randomAlphaOfLength(32)); + } if (randomBoolean()) { threadContext.putHeader( AuditTrail.X_FORWARDED_FOR_HEADER, @@ -1306,6 +1309,7 @@ public void testAnonymousAccessDeniedTransport() throws Exception { indicesRequest(request, checkedFields, checkedArrayFields); restOrTransportOrigin(request, threadContext, checkedFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -1341,6 +1345,7 @@ public void testAnonymousAccessDeniedRest() throws Exception { checkedFields.put(LoggingAuditTrail.REQUEST_BODY_FIELD_NAME, expectedMessage); } opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map()); @@ -1373,6 +1378,7 @@ public void testAuthenticationFailed() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -1400,6 +1406,7 @@ public void testAuthenticationFailedNoToken() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -1447,6 +1454,7 @@ public void testAuthenticationFailedRest() throws Exception { checkedFields.put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, "foo=bar"); } opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map()); @@ -1489,6 +1497,7 @@ public void testAuthenticationFailedRestNoToken() throws Exception { checkedFields.put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, "bar=baz"); } opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map()); @@ -1526,6 +1535,7 @@ public void testAuthenticationFailedRealm() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); } @@ -1570,6 +1580,7 @@ public void testAuthenticationFailedRealmRest() throws Exception { checkedFields.put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, "_param=baz"); } opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map()); } @@ -1595,6 +1606,7 @@ public void testAccessGranted() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -1615,6 +1627,7 @@ public void testAccessGranted() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -1719,6 +1732,7 @@ public void testSystemAccessGranted() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); clearLog(); @@ -1746,6 +1760,7 @@ public void testSystemAccessGranted() throws Exception { authentication(authentication, checkedFields); restOrTransportOrigin(request, threadContext, checkedFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); if (index != null) { checkedArrayFields.put(LoggingAuditTrail.INDICES_FIELD_NAME, new String[] { index }); @@ -1782,6 +1797,7 @@ public void testAccessGrantedInternalSystemAction() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); } @@ -1806,6 +1822,7 @@ public void testAccessGrantedInternalSystemActionNonSystemUser() throws Exceptio restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -1826,6 +1843,7 @@ public void testAccessGrantedInternalSystemActionNonSystemUser() throws Exceptio restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -1863,6 +1881,7 @@ public void testAccessDenied() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -1883,6 +1902,7 @@ public void testAccessDenied() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -1922,6 +1942,7 @@ public void testTamperedRequestRest() throws Exception { checkedFields.put(LoggingAuditTrail.URL_QUERY_FIELD_NAME, "_param=baz"); } opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map()); @@ -1949,6 +1970,7 @@ public void testTamperedRequest() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -1978,6 +2000,7 @@ public void testTamperedRequestWithUser() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -1997,6 +2020,7 @@ public void testTamperedRequestWithUser() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -2028,6 +2052,7 @@ public void testConnectionDenied() throws Exception { .put(LoggingAuditTrail.TRANSPORT_PROFILE_FIELD_NAME, profile) .put(LoggingAuditTrail.RULE_FIELD_NAME, "deny _all"); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map()); @@ -2066,6 +2091,7 @@ public void testConnectionGranted() throws Exception { .put(LoggingAuditTrail.TRANSPORT_PROFILE_FIELD_NAME, profile) .put(LoggingAuditTrail.RULE_FIELD_NAME, "allow default:accept_all"); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map()); } @@ -2097,6 +2123,7 @@ public void testRunAsGranted() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -2134,6 +2161,7 @@ public void testRunAsDenied() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -2185,6 +2213,7 @@ public void testAuthenticationSuccessRest() throws Exception { } authentication(authentication, checkedFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map()); @@ -2210,6 +2239,7 @@ public void testAuthenticationSuccessRest() throws Exception { } authentication(authentication, checkedFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map()); } @@ -2238,6 +2268,7 @@ public void testAuthenticationSuccessTransport() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); @@ -2257,6 +2288,7 @@ public void testAuthenticationSuccessTransport() throws Exception { restOrTransportOrigin(request, threadContext, checkedFields); indicesRequest(request, checkedFields, checkedArrayFields); opaqueId(threadContext, checkedFields); + traceId(threadContext, checkedFields); forwardedFor(threadContext, checkedFields); assertMsg(logger, checkedFields.map(), checkedArrayFields.map()); } @@ -2664,16 +2696,31 @@ private static void authentication(Authentication authentication, MapBuilder checkedFields) { - final String opaqueId = threadContext.getHeader(Task.X_OPAQUE_ID_HTTP_HEADER); - if (opaqueId != null) { - checkedFields.put(LoggingAuditTrail.OPAQUE_ID_FIELD_NAME, opaqueId); - } + setFieldFromThreadContext(threadContext, checkedFields, Task.X_OPAQUE_ID_HTTP_HEADER, LoggingAuditTrail.OPAQUE_ID_FIELD_NAME); + } + + private static void traceId(ThreadContext threadContext, MapBuilder checkedFields) { + setFieldFromThreadContext(threadContext, checkedFields, Task.TRACE_ID, LoggingAuditTrail.TRACE_ID_FIELD_NAME); } private static void forwardedFor(ThreadContext threadContext, MapBuilder checkedFields) { - final String forwardedFor = threadContext.getHeader(AuditTrail.X_FORWARDED_FOR_HEADER); - if (forwardedFor != null) { - checkedFields.put(LoggingAuditTrail.X_FORWARDED_FOR_FIELD_NAME, forwardedFor); + setFieldFromThreadContext( + threadContext, + checkedFields, + AuditTrail.X_FORWARDED_FOR_HEADER, + LoggingAuditTrail.X_FORWARDED_FOR_FIELD_NAME + ); + } + + private static void setFieldFromThreadContext( + ThreadContext threadContext, + MapBuilder checkedFields, + String threadContextFieldName, + String logFieldName + ) { + final String value = threadContext.getHeader(threadContextFieldName); + if (value != null) { + checkedFields.put(logFieldName, value); } } From d172941755ccc8bbe64eaf2d1a44f950e06a27fe Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Thu, 20 Jan 2022 12:11:12 +0100 Subject: [PATCH 3/8] Update docs/changelog/82849.yaml --- docs/changelog/82849.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/82849.yaml diff --git a/docs/changelog/82849.yaml b/docs/changelog/82849.yaml new file mode 100644 index 0000000000000..6f50897ee902d --- /dev/null +++ b/docs/changelog/82849.yaml @@ -0,0 +1,5 @@ +pr: 82849 +summary: Emit `trace.id` into audit logs +area: "Audit, Infra/Logging" +type: enhancement +issues: [] From f93e14e06ded2f75cf0316488d650c1dee592d67 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 20 Jan 2022 12:12:08 +0100 Subject: [PATCH 4/8] unused method --- .../security/audit/logfile/LoggingAuditTrail.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index c2b35a55c08b5..0d9d820f4bfa7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -1453,18 +1453,6 @@ LogEntryBuilder withRequestId(String requestId) { return this; } - LogEntryBuilder withXOpaqueIdAndTraceId(ThreadContext threadContext) { - final String opaqueId = threadContext.getHeader(Task.X_OPAQUE_ID_HTTP_HEADER); - if (opaqueId != null) { - logEntry.with(OPAQUE_ID_FIELD_NAME, opaqueId); - } - final String traceId = threadContext.getHeader(Task.TRACE_ID); - if (traceId != null) { - logEntry.with(TRACE_ID_FIELD_NAME, opaqueId); - } - return this; - } - LogEntryBuilder withThreadContext(ThreadContext threadContext) { setThreadContextField(threadContext, AuditTrail.X_FORWARDED_FOR_HEADER, X_FORWARDED_FOR_FIELD_NAME); setThreadContextField(threadContext, Task.X_OPAQUE_ID_HTTP_HEADER, OPAQUE_ID_FIELD_NAME); From 8d8f9fb2e61d1ecd123ff399a50bebb3aeb6b13a Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 20 Jan 2022 12:24:20 +0100 Subject: [PATCH 5/8] only one area --- docs/changelog/82849.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/82849.yaml b/docs/changelog/82849.yaml index 6f50897ee902d..4b3bdcc16fac2 100644 --- a/docs/changelog/82849.yaml +++ b/docs/changelog/82849.yaml @@ -1,5 +1,5 @@ pr: 82849 summary: Emit `trace.id` into audit logs -area: "Audit, Infra/Logging" +area: "Audit" type: enhancement issues: [] From 891f6c2c398cb7596150bd178e2a592d6c471dc5 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 25 Jan 2022 10:06:08 +0100 Subject: [PATCH 6/8] add docs --- x-pack/docs/en/security/auditing/event-types.asciidoc | 10 +++++++--- x-pack/plugin/core/src/main/config/log4j2.properties | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index 9601ac0090262..1c1abac9d05ac 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -6,7 +6,7 @@ When you are <>, a single client request might generate multiple audit events, across multiple cluster nodes. The common `request.id` attribute can be used to correlate the associated events. -Use the <> +Use the <> setting in `elasticsearch.yml` to specify the kind of events you want to include in the auditing output. @@ -534,8 +534,8 @@ The following list shows attributes that are common to all audit event types: In addition, if `event.type` equals <>, the `event.action` attribute takes one of the following values: `put_user`, `change_password`, `put_role`, `put_role_mapping`, -`change_enable_user`, `change_disable_user`, `put_privileges`, `create_apikey`, -`delete_user`, `delete_role`, `delete_role_mapping`, `invalidate_apikeys` or +`change_enable_user`, `change_disable_user`, `put_privileges`, `create_apikey`, +`delete_user`, `delete_role`, `delete_role_mapping`, `invalidate_apikeys` or `delete_privileges`. `request.id` :: A synthetic identifier that can be used to correlate the events @@ -560,6 +560,10 @@ show more details about the requesting client: the request associated with this event. This header can be used freely by the client to mark API calls, as it has no semantics in Elasticsearch. +`trace_id` :: The identifier extracted from the `traceparent` HTTP header + (if present) of the request associated with this event. + It allows to surface audit logs into the Trace Logs feature + of Elastic APM. `x_forwarded_for` :: The verbatim value of the `X-Forwarded-For` HTTP request header (if present) of the request associated with the audit event. This header is commonly added by proxies diff --git a/x-pack/plugin/core/src/main/config/log4j2.properties b/x-pack/plugin/core/src/main/config/log4j2.properties index a4ed55d8ea778..6c1d04596246e 100644 --- a/x-pack/plugin/core/src/main/config/log4j2.properties +++ b/x-pack/plugin/core/src/main/config/log4j2.properties @@ -77,6 +77,7 @@ appender.audit_rolling.layout.pattern = {\ # "request.name" if the event is in connection to a transport message this is the name of the request class, similar to how rest requests are identified by the url path (internal) # "indices" the array of indices that the "action" is acting upon # "opaque_id" opaque value conveyed by the "X-Opaque-Id" request header +# "trace_id" an identifier conveyed by the part of "traceparent" request header # "x_forwarded_for" the addresses from the "X-Forwarded-For" request header, as a verbatim string value (not an array) # "transport.profile" name of the transport profile in case this is a "connection_granted" or "connection_denied" event # "rule" name of the applied rule if the "origin.type" is "ip_filter" From 1d37f5527823986e049b2d9ac3c9550833ca6fb4 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 25 Jan 2022 10:14:05 +0100 Subject: [PATCH 7/8] update x-opaque-id --- x-pack/docs/en/security/auditing/event-types.asciidoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index 1c1abac9d05ac..1dd7873abc100 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -558,8 +558,9 @@ show more details about the requesting client: or `local_node` (the local node issued the request). `opaque_id` :: The value of the `X-Opaque-Id` HTTP header (if present) of the request associated with this event. This header can - be used freely by the client to mark API calls, as it has - no semantics in Elasticsearch. + be used freely by the client to mark API calls, but value + should be from a finite set. + See more: <> `trace_id` :: The identifier extracted from the `traceparent` HTTP header (if present) of the request associated with this event. It allows to surface audit logs into the Trace Logs feature From 653fd1367ede4bf423baf9f0a56ed6d9c79aee4d Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 25 Jan 2022 10:38:44 +0100 Subject: [PATCH 8/8] tweak doc --- x-pack/docs/en/security/auditing/event-types.asciidoc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/x-pack/docs/en/security/auditing/event-types.asciidoc b/x-pack/docs/en/security/auditing/event-types.asciidoc index 1dd7873abc100..2f92ee66c0c40 100644 --- a/x-pack/docs/en/security/auditing/event-types.asciidoc +++ b/x-pack/docs/en/security/auditing/event-types.asciidoc @@ -557,9 +557,7 @@ show more details about the requesting client: `transport` (request was received on the transport channel), or `local_node` (the local node issued the request). `opaque_id` :: The value of the `X-Opaque-Id` HTTP header (if present) of - the request associated with this event. This header can - be used freely by the client to mark API calls, but value - should be from a finite set. + the request associated with this event. See more: <> `trace_id` :: The identifier extracted from the `traceparent` HTTP header (if present) of the request associated with this event.