From 5f1dce0d2228fe66f2155765856d5d5f32ccf66d Mon Sep 17 00:00:00 2001 From: Ivan Senic Date: Mon, 30 Mar 2020 14:34:18 +0200 Subject: [PATCH 1/5] closes #637: exposing count of reported resource timing elements --- .../beacon/recorder/BeaconRecorder.java | 18 + .../ResourceTimingBeaconRecorder.java | 346 ++++++++++++++++++ .../model/EumServerConfiguration.java | 7 + .../model/ResourceTimingSettings.java | 18 + .../server/metrics/BeaconMetricManager.java | 13 + .../src/main/resources/application.yml | 4 + .../ResourceTimingBeaconRecorderTest.java | 188 ++++++++++ .../metrics/BeaconMetricManagerTest.java | 42 ++- 8 files changed, 633 insertions(+), 3 deletions(-) create mode 100644 components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/BeaconRecorder.java create mode 100644 components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorder.java create mode 100644 components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/ResourceTimingSettings.java create mode 100644 components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java diff --git a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/BeaconRecorder.java b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/BeaconRecorder.java new file mode 100644 index 0000000000..49b575908e --- /dev/null +++ b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/BeaconRecorder.java @@ -0,0 +1,18 @@ +package rocks.inspectit.oce.eum.server.beacon.recorder; + +import rocks.inspectit.oce.eum.server.beacon.Beacon; + +/** + * Interface for all components acting as {@link BeaconRecorder}. + * BeaconRecorder are intended to record custom complicated metrics from a fully-processed Beacon. + */ +public interface BeaconRecorder { + + /** + * Records arbitrary metrics from given {@link Beacon} + * + * @param beacon Fully-processed beacon + */ + void record(Beacon beacon); + +} diff --git a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorder.java b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorder.java new file mode 100644 index 0000000000..8d8b1b35c4 --- /dev/null +++ b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorder.java @@ -0,0 +1,346 @@ +package rocks.inspectit.oce.eum.server.beacon.recorder; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.opencensus.common.Scope; +import lombok.Builder; +import lombok.RequiredArgsConstructor; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.http.HttpRequest; +import org.springframework.lang.Nullable; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; +import rocks.inspectit.oce.eum.server.beacon.Beacon; +import rocks.inspectit.oce.eum.server.metrics.MeasuresAndViewsManager; +import rocks.inspectit.ocelot.config.model.metrics.definition.MetricDefinitionSettings; +import rocks.inspectit.ocelot.config.model.metrics.definition.ViewDefinitionSettings; + +import javax.annotation.PostConstruct; +import java.io.IOException; +import java.util.*; +import java.util.stream.Stream; + +/** + * This {@link BeaconRecorder} processes plain resource timing entry from the {@link Beacon} and exposes metric that: + * + *

+ * The impl depends heavily on the Boomerang compression of the resource timing entries in the beacon. Please read + * ResourceTiming + * Boomerang documentation first. + */ +@Component +@ConditionalOnProperty(value = "inspectit-eum-server.resource-timing.enabled", havingValue = "true") +@RequiredArgsConstructor +@Slf4j +public class ResourceTimingBeaconRecorder implements BeaconRecorder { + + /** + * Metric definition for the resource timing total metric. + */ + private static final MetricDefinitionSettings RESOURCE_TIMING_TOTAL; + + static { + Map tags = new HashMap<>(); + tags.put("initiatorType", true); + tags.put("cached", true); + tags.put("crossOrigin", true); + + RESOURCE_TIMING_TOTAL = MetricDefinitionSettings.builder() + .type(MetricDefinitionSettings.MeasureType.LONG) + .unit("ms") + .view("resource_timing_total/COUNT", ViewDefinitionSettings.builder() + .tags(tags) + .aggregation(ViewDefinitionSettings.Aggregation.COUNT) + .build() + ) + .build(); + } + + /** + * Object mapper used to read the resource timing + */ + @Autowired + private final ObjectMapper objectMapper; + + /** + * {@link MeasuresAndViewsManager} for exposing metrics. + */ + @Autowired + private final MeasuresAndViewsManager measuresAndViewsManager; + + /** + * Init metric(s). + */ + @PostConstruct + public void initMetric() { + measuresAndViewsManager.updateMetrics("resource_timing_total", RESOURCE_TIMING_TOTAL); + } + + /** + * {@inheritDoc} + *

+ * Parses the restiming entry from the beacon and exposes metric(s) about resource timing if parsing is + * success. + */ + @Override + public void record(Beacon beacon) { + // this is the URL where the resources have been loaded + String url = beacon.get("u"); + + Optional.ofNullable(beacon.get("restiming")) + .ifPresent(resourceTiming -> this.resourceTimingResults(resourceTiming) + .forEach(rs -> this.record(rs, url)) + ); + } + + /** + * Records one {@link ResourceTimingEntry} to the exposed metric(s). + * + * @param resourceTimingEntry entry + * @param url URL of the page where the resource has been loaded from. + */ + private void record(ResourceTimingEntry resourceTimingEntry, String url) { + Map extra = new HashMap<>(); + boolean sameOrigin = isSameOrigin(url, resourceTimingEntry.url); + extra.put("crossOrigin", String.valueOf(!sameOrigin)); + extra.put("initiatorType", resourceTimingEntry.getInitiatorType().toString()); + // TODO is this OK, only setting cached if it's same origin, otherwise we can not know + if (sameOrigin) { + extra.put("cached", String.valueOf(resourceTimingEntry.isCached(true))); + } + + try (Scope scope = measuresAndViewsManager.getTagContext(extra).buildScoped()) { + // TODO I think we should already collect there the load time, thus we would have a COUNT and a SUM of time + // we would have most of the stuff needed then + // and it would make tests more reliable then now + measuresAndViewsManager.recordMeasure("resource_timing_total", RESOURCE_TIMING_TOTAL, 1L); + } + } + + /** + * Takes Boomerang resource timing JSON and returns stream of found {@link ResourceTimingEntry}s. + * + * @param resourceTiming json + * @return stream + */ + private Stream resourceTimingResults(String resourceTiming) { + JsonNode rootNode; + try { + rootNode = objectMapper.readTree(resourceTiming); + } catch (IOException e) { + log.error("Error converting resource timing json to tree.", e); + return Stream.empty(); + } + + return findAllTimingValuesAsText(rootNode).entrySet() + .stream() + .flatMap(entry -> this.resolveResourceTimingStringValue(entry.getKey(), entry.getValue())); + + } + + /** + * Helper to construct map of resource URLs to resource timing details. + * + * @param node root node + * @return Map where keys are resource URLs and values are the Boomerang compressed resource timing string. + */ + private Map findAllTimingValuesAsText(JsonNode node) { + Map map = new HashMap<>(); + this.findAllTimingValuesAsText(node, map, ""); + return map; + } + + private void findAllTimingValuesAsText(JsonNode node, Map foundSoFar, String prefix) { + if (node.isValueNode()) { + if (node.isTextual()) { + foundSoFar.put(prefix, node.textValue()); + } + } else { + node.fields().forEachRemaining(entry -> this.findAllTimingValuesAsText(entry.getValue(), foundSoFar, prefix + entry.getKey())); + } + + } + + /** + * Maps one resource URL and it's timing given as the Boomerang compressed string to the stream of {@link ResourceTimingEntry}. + *

+ * Note that one compressed string can contain multiple loads of the same entry. This method ignores any additional + * data (transfer size, image size, etc) from the Boomerang string. + * + * @param url URL of the loaded resource. + * @param value Boomerang compressed resource timing string for a single entry + * @return + */ + private Stream resolveResourceTimingStringValue(String url, String value) { + return Stream.of(value) + // split by pipe, as pipe separates same resource load times if executed more than once + .flatMap(possibleMultipleValue -> { + String[] split = possibleMultipleValue.split("\\|"); + return Arrays.stream(split); + }) + .flatMap(singeValue -> ResourceTimingEntry.from(url, singeValue).map(Stream::of).orElseGet(Stream::empty)); + } + + /** + * Checks if two URLs are considered as same origin. Based on {@link org.springframework.web.util.WebUtils#isSameOrigin(HttpRequest)}. + * + * @param u1 first url + * @param u2 second url + * @return Returns true if two urls are considered as same-origin + * @see org.springframework.web.util.WebUtils#isSameOrigin(HttpRequest) + * @see 'https://developer.mozilla.org/en-US/docs/Web/Security/Same-origin_policy' + */ + private static boolean isSameOrigin(String u1, String u2) { + UriComponents uriComponents1 = UriComponentsBuilder.fromUriString(u1).build(); + UriComponents uriComponents2 = UriComponentsBuilder.fromUriString(u2).build(); + + return Objects.equals(uriComponents1.getScheme(), uriComponents2.getScheme()) && + Objects.equals(uriComponents1.getHost(), uriComponents2.getHost()) && + getPort(uriComponents1.getScheme(), uriComponents1.getPort()) == getPort(uriComponents2.getScheme(), uriComponents2.getPort()); + } + + private static int getPort(@Nullable String scheme, int port) { + if (port == -1) { + if ("http".equals(scheme) || "ws".equals(scheme)) { + port = 80; + } else if ("https".equals(scheme) || "wss".equals(scheme)) { + port = 443; + } + } + return port; + } + + @Value + @Builder + public static class ResourceTimingEntry { + + /** + * Url of the resource. + */ + String url; + + /** + * Initiator type. + */ + InitiatorType initiatorType; + + /** + * Timings array in following order: + *
+ * timings = "[startTime],[responseEnd],[responseStart],[requestStart],[connectEnd],[secureConnectionStart],[connectStart],[domainLookupEnd],[domainLookupStart],[redirectEnd],[redirectStart]" + *

+ * Note that array does not need to be fully populated if entries from the end are missing. + */ + Integer[] timings; + + /** + * Cached resources only have 2 timing values if considered as same-origin requests. + * + * @param sameOrigin If this is considered as same origin resource loading + * @return If cached or not. + */ + public boolean isCached(boolean sameOrigin) { + if (this.timings == null || this.timings.length < 3) { + return sameOrigin; + } + return false; + } + + /** + * Constructs the {@link ResourceTimingEntry} from a single (non-piped) string value. + *

+ * Will resolve to empty if it contains only additional data. + * + * @return ResourceTimingEntry as optional + */ + public static Optional from(String url, String value) { + // check if this string contains additional data + // if so cut it from processing + String toProcess = value; + int additionalDataIndex = value.indexOf('*'); + if (additionalDataIndex > -1) { + toProcess = value.substring(0, additionalDataIndex); + } + + if (StringUtils.isEmpty(toProcess)) { + return Optional.empty(); + } + + // initiator is always first char + InitiatorType initiatorType = InitiatorType.from(toProcess.charAt(0)); + + // then split by comma to get all timings + String[] timingsAsBase36Strings = toProcess.substring(1).split(","); + + // then convert timings in base36 to int values + // if empty then it's zero + Integer[] timings = Arrays.stream(timingsAsBase36Strings) + .map(v -> StringUtils.isEmpty(v) ? 0 : Integer.parseInt(v, 36)) + .toArray(Integer[]::new); + + // then build the entry + return Optional.of(ResourceTimingEntry.builder() + .url(url) + .initiatorType((initiatorType)) + .timings(timings) + .build() + ); + } + + } + + /** + * Initiator type represented by a char. + * + * @see 'https://developer.akamai.com/tools/boomerang/docs/BOOMR.plugins.ResourceTiming.html' + */ + public enum InitiatorType { + OTHER('0'), + IMG('1'), + LINK('2'), + SCRIPT('3'), + CSS('4'), + XML_HTTP_REQUEST('5'), + HTML('6'), + IMAGE('7'), + BEACON('8'), + FETCH('9'), + IFRAME('a'), + BODY('b'), + INPUT('c'), + OBJECT('d'), + VIDEO('e'), + AUDIO('f'), + SOURCE('g'), + TRACK('h'), + EMBED('i'), + EVENT_SOURCE('j'); + + private char identifier; + + InitiatorType(char identifier) { + this.identifier = identifier; + } + + /** + * Returns the {@link InitiatorType} represented by this char. If not found {@link #OTHER} is returned. + * + * @param c identifier + * @return {@link InitiatorType} + */ + public static InitiatorType from(char c) { + return Arrays.stream(InitiatorType.values()) + .filter(initiatorType -> initiatorType.identifier == c) + .findFirst() + .orElse(OTHER); + } + + } +} diff --git a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/EumServerConfiguration.java b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/EumServerConfiguration.java index 0e2e224e4b..ebf706cb5f 100644 --- a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/EumServerConfiguration.java +++ b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/EumServerConfiguration.java @@ -45,4 +45,11 @@ public class EumServerConfiguration { */ @Valid private ExportersSettings exporters; + + /** + * The resource timing settings. + */ + @Valid + private ResourceTimingSettings resourceTiming; + } diff --git a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/ResourceTimingSettings.java b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/ResourceTimingSettings.java new file mode 100644 index 0000000000..2894a1603e --- /dev/null +++ b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/ResourceTimingSettings.java @@ -0,0 +1,18 @@ +package rocks.inspectit.oce.eum.server.configuration.model; + +import lombok.Data; +import org.springframework.validation.annotation.Validated; + +/** + * Resource timing settings. + */ +@Data +@Validated +public class ResourceTimingSettings { + + /** + * If resource timing is enabled or not. + */ + private boolean enabled = true; + +} diff --git a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/metrics/BeaconMetricManager.java b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/metrics/BeaconMetricManager.java index b966780fae..0696f35086 100644 --- a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/metrics/BeaconMetricManager.java +++ b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/metrics/BeaconMetricManager.java @@ -7,13 +7,16 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; import rocks.inspectit.oce.eum.server.arithmetic.RawExpression; import rocks.inspectit.oce.eum.server.beacon.Beacon; +import rocks.inspectit.oce.eum.server.beacon.recorder.BeaconRecorder; import rocks.inspectit.oce.eum.server.configuration.model.BeaconMetricDefinitionSettings; import rocks.inspectit.oce.eum.server.configuration.model.BeaconRequirement; import rocks.inspectit.oce.eum.server.configuration.model.EumServerConfiguration; import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -29,6 +32,9 @@ public class BeaconMetricManager { @Autowired private MeasuresAndViewsManager measuresAndViewsManager; + @Autowired(required = false) + private List beaconRecorders; + /** * Maps metric definitions to expressions. */ @@ -55,6 +61,13 @@ public boolean processBeacon(Beacon beacon) { } } + // allow each beacon recorder to record stuff + if (!CollectionUtils.isEmpty(beaconRecorders)) { + try (Scope scope = getTagContextForBeacon(beacon).buildScoped()) { + beaconRecorders.forEach(beaconRecorder -> beaconRecorder.record(beacon)); + } + } + return successful; } diff --git a/components/inspectit-ocelot-eum-server/src/main/resources/application.yml b/components/inspectit-ocelot-eum-server/src/main/resources/application.yml index 940e40efab..1213f5224e 100644 --- a/components/inspectit-ocelot-eum-server/src/main/resources/application.yml +++ b/components/inspectit-ocelot-eum-server/src/main/resources/application.yml @@ -133,6 +133,10 @@ inspectit-eum-server: tags: is_error: true + # settings for exposing resource timing metrics + resource-timing: + enabled: true + # ACTUATOR PROPERTIES management: # Whether to enable or disable all endpoints by default. diff --git a/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java b/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java new file mode 100644 index 0000000000..305acf8854 --- /dev/null +++ b/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java @@ -0,0 +1,188 @@ +package rocks.inspectit.oce.eum.server.beacon.recorder; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.opencensus.tags.Tags; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import rocks.inspectit.oce.eum.server.beacon.Beacon; +import rocks.inspectit.oce.eum.server.metrics.MeasuresAndViewsManager; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class ResourceTimingBeaconRecorderTest { + + ResourceTimingBeaconRecorder recorder; + + @Mock + MeasuresAndViewsManager measuresAndViewsManager; + + ObjectMapper objectMapper; + + @BeforeEach + public void init() { + lenient().when(measuresAndViewsManager.getTagContext(any())).thenReturn(Tags.getTagger().emptyBuilder()); + + objectMapper = new ObjectMapper(); + recorder = new ResourceTimingBeaconRecorder(objectMapper, measuresAndViewsManager); + } + + @Nested + class Record { + + @Captor + ArgumentCaptor> tagsCaptor; + + @Test + public void noResourceTimingInfo() { + Beacon beacon = Beacon.of(Collections.emptyMap()); + + recorder.record(beacon); + + verifyNoMoreInteractions(measuresAndViewsManager); + } + + @Test + public void notAJson() { + Beacon beacon = Beacon.of(Collections.singletonMap("restiming", "This is not a valid json")); + + recorder.record(beacon); + + verifyNoMoreInteractions(measuresAndViewsManager); + } + + @Test + public void nestedResources() { + String json = "" + + "{\n" + + " \"http\": {\n" + + " \"://myhost/\": {\n" + + " \"boomerang/plugins/\": {\n" + + " \"r\": {\n" + + " \"t.js\": \"32o,2u,2q,25*1d67,9s,wdu*20\",\n" + + " \"estiming.js\": \"02p,2w,2p,24*1efk,9z,y2i*20\"\n" + + " }\n" + + " }\n" + + " },\n" + + " \"s://www.google.de/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png\": \"*02k,7k,1y,a2|180,3l\"\n" + + " }\n" + + "}"; + Map fields = new HashMap<>(); + fields.put("u", "http://myhost/somepage.html"); + fields.put("restiming", json); + Beacon beacon = Beacon.of(fields); + + recorder.record(beacon); + + verify(measuresAndViewsManager, atLeastOnce()).getTagContext(tagsCaptor.capture()); + verify(measuresAndViewsManager, times(3)).recordMeasure(eq("resource_timing_total"), any(), eq(1L)); + verifyNoMoreInteractions(measuresAndViewsManager); + + assertThat(tagsCaptor.getAllValues()).hasSize(3) + // t.js + .anySatisfy(map -> assertThat(map).hasSize(3) + .containsEntry("initiatorType", "SCRIPT") + .containsEntry("crossOrigin", "false") + .containsEntry("cached", "false") + ) + // estiming.js + .anySatisfy(map -> assertThat(map).hasSize(3) + .containsEntry("initiatorType", "OTHER") + .containsEntry("crossOrigin", "false") + .containsEntry("cached", "false") + ) + // googlelogo_color_272x92dp.png + .anySatisfy(map -> assertThat(map).hasSize(2) + .containsEntry("initiatorType", "IMG") + .containsEntry("crossOrigin", "true") + ); + + } + + @Test + public void pipedData() { + // intentionally change the initiator + String json = "" + + "{\n" + + " \"https://www.google.de/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png\": \"*02k,7k,1y,a2|180,3l|280,3l\"\n" + + "}"; + Map fields = new HashMap<>(); + fields.put("u", "http://myhost/somepage.html"); + fields.put("restiming", json); + Beacon beacon = Beacon.of(fields); + recorder.record(beacon); + + verify(measuresAndViewsManager, atLeastOnce()).getTagContext(tagsCaptor.capture()); + verify(measuresAndViewsManager, times(2)).recordMeasure(eq("resource_timing_total"), any(), eq(1L)); + verifyNoMoreInteractions(measuresAndViewsManager); + + assertThat(tagsCaptor.getAllValues()).hasSize(2) + // first + .anySatisfy(map -> assertThat(map).hasSize(2) + .containsEntry("initiatorType", "IMG") + .containsEntry("crossOrigin", "true") + ) + // second + .anySatisfy(map -> assertThat(map).hasSize(2) + .containsEntry("initiatorType", "LINK") + .containsEntry("crossOrigin", "true") + ); + } + + @Test + public void onlyAdditionalData() { + // intentionally change the initiator + String json = "" + + "{\n" + + " \"https://www.google.de/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png\": \"*02k,7k,1y,a2\"\n" + + "}"; + Map fields = new HashMap<>(); + fields.put("u", "http://myhost/somepage.html"); + fields.put("restiming", json); + Beacon beacon = Beacon.of(fields); + recorder.record(beacon); + + verifyNoMoreInteractions(measuresAndViewsManager); + } + + @Test + public void wrongInitiator() { + // intentionally change the initiator + String json = "" + + "{\n" + + " \"https://www.google.de/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png\": \"w80,3l\"\n" + + "}"; + Map fields = new HashMap<>(); + fields.put("u", "http://myhost/somepage.html"); + fields.put("restiming", json); + Beacon beacon = Beacon.of(fields); + recorder.record(beacon); + + verify(measuresAndViewsManager, atLeastOnce()).getTagContext(tagsCaptor.capture()); + verify(measuresAndViewsManager, times(1)).recordMeasure(eq("resource_timing_total"), any(), eq(1L)); + verifyNoMoreInteractions(measuresAndViewsManager); + + assertThat(tagsCaptor.getAllValues()).hasSize(1) + // first + .anySatisfy(map -> assertThat(map).hasSize(2) + .containsEntry("initiatorType", "OTHER") + .containsEntry("crossOrigin", "true") + ); + } + + } + +} diff --git a/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/metrics/BeaconMetricManagerTest.java b/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/metrics/BeaconMetricManagerTest.java index f35469ffa7..906a152841 100644 --- a/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/metrics/BeaconMetricManagerTest.java +++ b/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/metrics/BeaconMetricManagerTest.java @@ -1,17 +1,21 @@ package rocks.inspectit.oce.eum.server.metrics; -import io.opencensus.stats.*; -import io.opencensus.tags.TagKey; +import com.google.common.collect.ImmutableList; +import io.opencensus.stats.MeasureMap; +import io.opencensus.stats.StatsRecorder; +import io.opencensus.stats.ViewManager; +import io.opencensus.tags.Tags; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Answers; -import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import rocks.inspectit.oce.eum.server.beacon.Beacon; +import rocks.inspectit.oce.eum.server.beacon.recorder.BeaconRecorder; import rocks.inspectit.oce.eum.server.configuration.model.BeaconMetricDefinitionSettings; import rocks.inspectit.oce.eum.server.configuration.model.EumServerConfiguration; import rocks.inspectit.oce.eum.server.configuration.model.EumTagsSettings; @@ -34,6 +38,9 @@ public class BeaconMetricManagerTest { @Mock EumServerConfiguration configuration; + @Mock + MeasuresAndViewsManager measuresAndViewsManager; + @Mock EumTagsSettings tagSettings; @@ -46,6 +53,11 @@ public class BeaconMetricManagerTest { @Mock MeasureMap measureMap; + @Spy + List beaconRecorders = ImmutableList.of( + mock(BeaconRecorder.class) + ); + @Nested class ProcessBeacon { @@ -75,6 +87,13 @@ void setupConfiguration() { definitionMap.put("Dummy metric name", dummyMetricDefinition); } + @BeforeEach + public void setupMocks() { + when(configuration.getTags()).thenReturn(tagSettings); + when(tagSettings.getBeacon()).thenReturn(Collections.emptyMap()); + when(measuresAndViewsManager.getTagContext()).thenReturn(Tags.getTagger().emptyBuilder()); + } + @Test void verifyNoViewIsGeneratedWithEmptyBeacon() { when(configuration.getDefinitions()).thenReturn(definitionMap); @@ -95,5 +114,22 @@ void verifyNoViewIsGeneratedWithFullBeacon() { verifyZeroInteractions(viewManager, statsRecorder); } + + @Test + void beaconRecordersProcessed() { + when(configuration.getDefinitions()).thenReturn(definitionMap); + HashMap beaconMap = new HashMap<>(); + beaconMap.put("fake_beacon_field", "12d"); + Beacon beacon = Beacon.of(beaconMap); + + beaconMetricManager.processBeacon(beacon); + + assertThat(beaconRecorders).allSatisfy(beaconRecorder -> { + verify(beaconRecorder).record(beacon); + verifyNoMoreInteractions(beaconRecorder); + }); + verifyZeroInteractions(viewManager, statsRecorder); + } + } } From 2f36614e8afcd1db282fda9aa39af9219a02f6af Mon Sep 17 00:00:00 2001 From: Ivan Senic Date: Mon, 6 Apr 2020 11:02:31 +0200 Subject: [PATCH 2/5] integration fixes #1 --- .../beacon/recorder/BeaconRecorder.java | 3 +- .../ResourceTimingBeaconRecorder.java | 102 ++++++++++-------- .../model/ResourceTimingSettings.java | 2 +- .../ResourceTimingBeaconRecorderTest.java | 11 +- 4 files changed, 70 insertions(+), 48 deletions(-) diff --git a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/BeaconRecorder.java b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/BeaconRecorder.java index 49b575908e..30702eb5d4 100644 --- a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/BeaconRecorder.java +++ b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/BeaconRecorder.java @@ -9,7 +9,8 @@ public interface BeaconRecorder { /** - * Records arbitrary metrics from given {@link Beacon} + * Records arbitrary metrics from given the {@link Beacon}. This method will be invoked within the scope where + * global tags are already added. * * @param beacon Fully-processed beacon */ diff --git a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorder.java b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorder.java index 8d8b1b35c4..f3bbe649c4 100644 --- a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorder.java +++ b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorder.java @@ -23,6 +23,7 @@ import javax.annotation.PostConstruct; import java.io.IOException; import java.util.*; +import java.util.function.Function; import java.util.stream.Stream; /** @@ -42,9 +43,14 @@ public class ResourceTimingBeaconRecorder implements BeaconRecorder { /** - * Metric definition for the resource timing total metric. + * Name of the metric */ - private static final MetricDefinitionSettings RESOURCE_TIMING_TOTAL; + public static final String RESOURCE_TIME_METRIC_NAME = "resource_time"; + + /** + * Metric definition for the resource timing metric. + */ + private static final MetricDefinitionSettings RESOURCE_TIME; static { Map tags = new HashMap<>(); @@ -52,10 +58,10 @@ public class ResourceTimingBeaconRecorder implements BeaconRecorder { tags.put("cached", true); tags.put("crossOrigin", true); - RESOURCE_TIMING_TOTAL = MetricDefinitionSettings.builder() + RESOURCE_TIME = MetricDefinitionSettings.builder() .type(MetricDefinitionSettings.MeasureType.LONG) .unit("ms") - .view("resource_timing_total/COUNT", ViewDefinitionSettings.builder() + .view(RESOURCE_TIME_METRIC_NAME + "/COUNT", ViewDefinitionSettings.builder() .tags(tags) .aggregation(ViewDefinitionSettings.Aggregation.COUNT) .build() @@ -80,7 +86,7 @@ public class ResourceTimingBeaconRecorder implements BeaconRecorder { */ @PostConstruct public void initMetric() { - measuresAndViewsManager.updateMetrics("resource_timing_total", RESOURCE_TIMING_TOTAL); + measuresAndViewsManager.updateMetrics(RESOURCE_TIME_METRIC_NAME, RESOURCE_TIME); } /** @@ -94,10 +100,10 @@ public void record(Beacon beacon) { // this is the URL where the resources have been loaded String url = beacon.get("u"); - Optional.ofNullable(beacon.get("restiming")) - .ifPresent(resourceTiming -> this.resourceTimingResults(resourceTiming) - .forEach(rs -> this.record(rs, url)) - ); + String resourceTimings = beacon.get("restiming"); + if (resourceTimings != null) { + decodeResourceTimings(resourceTimings).forEach(rs -> this.record(rs, url)); + } } /** @@ -120,7 +126,7 @@ private void record(ResourceTimingEntry resourceTimingEntry, String url) { // TODO I think we should already collect there the load time, thus we would have a COUNT and a SUM of time // we would have most of the stuff needed then // and it would make tests more reliable then now - measuresAndViewsManager.recordMeasure("resource_timing_total", RESOURCE_TIMING_TOTAL, 1L); + measuresAndViewsManager.recordMeasure("resource_time", RESOURCE_TIME, 1L); } } @@ -130,7 +136,7 @@ private void record(ResourceTimingEntry resourceTimingEntry, String url) { * @param resourceTiming json * @return stream */ - private Stream resourceTimingResults(String resourceTiming) { + private Stream decodeResourceTimings(String resourceTiming) { JsonNode rootNode; try { rootNode = objectMapper.readTree(resourceTiming); @@ -139,7 +145,7 @@ private Stream resourceTimingResults(String resourceTiming) return Stream.empty(); } - return findAllTimingValuesAsText(rootNode).entrySet() + return flattenUrlTrie(rootNode).entrySet() .stream() .flatMap(entry -> this.resolveResourceTimingStringValue(entry.getKey(), entry.getValue())); @@ -151,7 +157,7 @@ private Stream resourceTimingResults(String resourceTiming) * @param node root node * @return Map where keys are resource URLs and values are the Boomerang compressed resource timing string. */ - private Map findAllTimingValuesAsText(JsonNode node) { + private Map flattenUrlTrie(JsonNode node) { Map map = new HashMap<>(); this.findAllTimingValuesAsText(node, map, ""); return map; @@ -163,9 +169,11 @@ private void findAllTimingValuesAsText(JsonNode node, Map foundS foundSoFar.put(prefix, node.textValue()); } } else { - node.fields().forEachRemaining(entry -> this.findAllTimingValuesAsText(entry.getValue(), foundSoFar, prefix + entry.getKey())); + // note the | (pipe) keys + // if a resource's URL is a prefix of another resource, then it terminates with a pipe symbol (|). + Function pipeResolver = (s) -> "|".equals(s) ? "" : s; + node.fields().forEachRemaining(entry -> this.findAllTimingValuesAsText(entry.getValue(), foundSoFar, prefix + pipeResolver.apply(entry.getKey()))); } - } /** @@ -261,37 +269,43 @@ public boolean isCached(boolean sameOrigin) { * @return ResourceTimingEntry as optional */ public static Optional from(String url, String value) { - // check if this string contains additional data - // if so cut it from processing - String toProcess = value; - int additionalDataIndex = value.indexOf('*'); - if (additionalDataIndex > -1) { - toProcess = value.substring(0, additionalDataIndex); - } - - if (StringUtils.isEmpty(toProcess)) { + try { + // check if this string contains additional data + // if so cut it from processing + String toProcess = value; + int additionalDataIndex = value.indexOf('*'); + if (additionalDataIndex > -1) { + toProcess = value.substring(0, additionalDataIndex); + } + + if (StringUtils.isEmpty(toProcess)) { + return Optional.empty(); + } + + // initiator is always first char + InitiatorType initiatorType = InitiatorType.from(toProcess.charAt(0)); + + // then split by comma to get all timings + String[] timingsAsBase36Strings = toProcess.substring(1).split(","); + + // then convert timings in base36 to int values + // if empty then it's zero + Integer[] timings = Arrays.stream(timingsAsBase36Strings) + .map(v -> StringUtils.isEmpty(v) ? 0 : Integer.parseInt(v, 36)) + .toArray(Integer[]::new); + + // then build the entry + return Optional.of(ResourceTimingEntry.builder() + .url(url) + .initiatorType((initiatorType)) + .timings(timings) + .build() + ); + } catch (Exception e) { + // in case of any exception return the empty result here + log.warn("Unable to create a resource timing entry for the URL {} with the Boomerang value {}.", url, value); return Optional.empty(); } - - // initiator is always first char - InitiatorType initiatorType = InitiatorType.from(toProcess.charAt(0)); - - // then split by comma to get all timings - String[] timingsAsBase36Strings = toProcess.substring(1).split(","); - - // then convert timings in base36 to int values - // if empty then it's zero - Integer[] timings = Arrays.stream(timingsAsBase36Strings) - .map(v -> StringUtils.isEmpty(v) ? 0 : Integer.parseInt(v, 36)) - .toArray(Integer[]::new); - - // then build the entry - return Optional.of(ResourceTimingEntry.builder() - .url(url) - .initiatorType((initiatorType)) - .timings(timings) - .build() - ); } } diff --git a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/ResourceTimingSettings.java b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/ResourceTimingSettings.java index 2894a1603e..7fdf2fec1a 100644 --- a/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/ResourceTimingSettings.java +++ b/components/inspectit-ocelot-eum-server/src/main/java/rocks/inspectit/oce/eum/server/configuration/model/ResourceTimingSettings.java @@ -13,6 +13,6 @@ public class ResourceTimingSettings { /** * If resource timing is enabled or not. */ - private boolean enabled = true; + private boolean enabled; } diff --git a/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java b/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java index 305acf8854..d0794a965e 100644 --- a/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java +++ b/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java @@ -70,6 +70,7 @@ public void nestedResources() { "{\n" + " \"http\": {\n" + " \"://myhost/\": {\n" + + " \"|\": \"6,2\",\n" + " \"boomerang/plugins/\": {\n" + " \"r\": {\n" + " \"t.js\": \"32o,2u,2q,25*1d67,9s,wdu*20\",\n" + @@ -88,10 +89,16 @@ public void nestedResources() { recorder.record(beacon); verify(measuresAndViewsManager, atLeastOnce()).getTagContext(tagsCaptor.capture()); - verify(measuresAndViewsManager, times(3)).recordMeasure(eq("resource_timing_total"), any(), eq(1L)); + verify(measuresAndViewsManager, times(4)).recordMeasure(eq("resource_timing_total"), any(), eq(1L)); verifyNoMoreInteractions(measuresAndViewsManager); - assertThat(tagsCaptor.getAllValues()).hasSize(3) + assertThat(tagsCaptor.getAllValues()).hasSize(4) + // | + .anySatisfy(map -> assertThat(map).hasSize(3) + .containsEntry("initiatorType", "HTML") + .containsEntry("crossOrigin", "false") + .containsEntry("cached", "true") + ) // t.js .anySatisfy(map -> assertThat(map).hasSize(3) .containsEntry("initiatorType", "SCRIPT") From 5888416501cf0dfcfde728a273d9b52eec5e7da9 Mon Sep 17 00:00:00 2001 From: Ivan Senic Date: Mon, 6 Apr 2020 11:50:38 +0200 Subject: [PATCH 3/5] docs for the resource timing --- .../enduser-monitoring/eum-server-configuration.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/inspectit-ocelot-documentation/docs/enduser-monitoring/eum-server-configuration.md b/inspectit-ocelot-documentation/docs/enduser-monitoring/eum-server-configuration.md index 85d9bfe052..74a8f7ca82 100644 --- a/inspectit-ocelot-documentation/docs/enduser-monitoring/eum-server-configuration.md +++ b/inspectit-ocelot-documentation/docs/enduser-monitoring/eum-server-configuration.md @@ -277,6 +277,20 @@ inspectit-eum-server: - COUNTRY_CODE ``` +## Resource timings + +The EUM server can extract information about the resources timings which are reported as part of the Boomerang beacon field `restiming`. +The resource timing information is decompressed from the beacon and exposed as part of the `resource_time` metric. +This metric contains following tags: + +| Tag | Description | +| --- | --- | +| `initiatorType` | The type of the element initiating the loading of the resource. See [all initiator types](https://developer.akamai.com/tools/boomerang/docs/BOOMR.plugins.ResourceTiming.html#.INITIATOR_TYPES). | +| `crossOrigin` | If a resource loading is considered as cross-origin request. See [more information about CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS). | +| `cached` | If a resource was cached and loaded from the browser disk or memory storage. Note that cached tag will only be set for same-origin requests, as some resource timing metrics are restricted and will not be provided cross-origin unless the Timing-Allow-Origin header permits. | + +The resource timing processing is enabled by default and can be disabled by setting the property `inspectit-eum-server.resource-timing.enabled` to `false.` + ## Exporters The EUM server comes with the same Prometheus and InfluxDB exporter as the Ocelot agent. From 212b5b7ce9476af7e484bed17b3aafea27ac5cdf Mon Sep 17 00:00:00 2001 From: Ivan Senic Date: Mon, 6 Apr 2020 14:44:52 +0200 Subject: [PATCH 4/5] fixing failing test --- .../beacon/recorder/ResourceTimingBeaconRecorderTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java b/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java index d0794a965e..ad157e6ae9 100644 --- a/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java +++ b/components/inspectit-ocelot-eum-server/src/test/java/rocks/inspectit/oce/eum/server/beacon/recorder/ResourceTimingBeaconRecorderTest.java @@ -89,7 +89,7 @@ public void nestedResources() { recorder.record(beacon); verify(measuresAndViewsManager, atLeastOnce()).getTagContext(tagsCaptor.capture()); - verify(measuresAndViewsManager, times(4)).recordMeasure(eq("resource_timing_total"), any(), eq(1L)); + verify(measuresAndViewsManager, times(4)).recordMeasure(eq("resource_time"), any(), eq(1L)); verifyNoMoreInteractions(measuresAndViewsManager); assertThat(tagsCaptor.getAllValues()).hasSize(4) @@ -133,7 +133,7 @@ public void pipedData() { recorder.record(beacon); verify(measuresAndViewsManager, atLeastOnce()).getTagContext(tagsCaptor.capture()); - verify(measuresAndViewsManager, times(2)).recordMeasure(eq("resource_timing_total"), any(), eq(1L)); + verify(measuresAndViewsManager, times(2)).recordMeasure(eq("resource_time"), any(), eq(1L)); verifyNoMoreInteractions(measuresAndViewsManager); assertThat(tagsCaptor.getAllValues()).hasSize(2) @@ -179,7 +179,7 @@ public void wrongInitiator() { recorder.record(beacon); verify(measuresAndViewsManager, atLeastOnce()).getTagContext(tagsCaptor.capture()); - verify(measuresAndViewsManager, times(1)).recordMeasure(eq("resource_timing_total"), any(), eq(1L)); + verify(measuresAndViewsManager, times(1)).recordMeasure(eq("resource_time"), any(), eq(1L)); verifyNoMoreInteractions(measuresAndViewsManager); assertThat(tagsCaptor.getAllValues()).hasSize(1) From 75524a1df07cfc6f4d5efcc1b8834facf991c07f Mon Sep 17 00:00:00 2001 From: Marius Oehler Date: Tue, 7 Apr 2020 11:16:07 +0200 Subject: [PATCH 5/5] Changed headline capitalization --- .../docs/enduser-monitoring/eum-server-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inspectit-ocelot-documentation/docs/enduser-monitoring/eum-server-configuration.md b/inspectit-ocelot-documentation/docs/enduser-monitoring/eum-server-configuration.md index 74a8f7ca82..ef62a6c41b 100644 --- a/inspectit-ocelot-documentation/docs/enduser-monitoring/eum-server-configuration.md +++ b/inspectit-ocelot-documentation/docs/enduser-monitoring/eum-server-configuration.md @@ -277,7 +277,7 @@ inspectit-eum-server: - COUNTRY_CODE ``` -## Resource timings +## Resource Timings The EUM server can extract information about the resources timings which are reported as part of the Boomerang beacon field `restiming`. The resource timing information is decompressed from the beacon and exposed as part of the `resource_time` metric.