APPSERV-12 Common and dynamic trace store size via replicated maps

nucleus/payara-modules/hazelcast-bootstrap/src/main/java/fish/payara/nucleus/config/ClusteredConfig.java

@@ -0,0 +1,173 @@
+/*
+ * Copyright (c) 2020 Payara Foundation and/or its affiliates. All rights reserved.
+ *
+ * The contents of this file are subject to the terms of either the GNU
+ * General Public License Version 2 only ("GPL") or the Common Development
+ * and Distribution License("CDDL") (collectively, the "License").  You
+ * may not use this file except in compliance with the License.  You can
+ * obtain a copy of the License at
+ * https://github.com/payara/Payara/blob/master/LICENSE.txt
+ * See the License for the specific
+ * language governing permissions and limitations under the License.
+ *
+ * When distributing the software, include this License Header Notice in each
+ * file and include the License file at glassfish/legal/LICENSE.txt.
+ *
+ * GPL Classpath Exception:
+ * The Payara Foundation designates this particular file as subject to the "Classpath"
+ * exception as provided by the Payara Foundation in the GPL Version 2 section of the License
+ * file that accompanied this code.
+ *
+ * Modifications:
+ * If applicable, add the following below the License Header, with the fields
+ * enclosed by brackets [] replaced by your own identifying information:
+ * "Portions Copyright [year] [name of copyright owner]"
+ *
+ * Contributor(s):
+ * If you wish your version of this file to be governed by only the CDDL or
+ * only the GPL Version 2, indicate your decision by adding "[Contributor]
+ * elects to include this software in this distribution under the [CDDL or GPL
+ * Version 2] license."  If you don't indicate a single choice of license, a
+ * recipient has the option to distribute your version of this file under
+ * either the CDDL, the GPL Version 2 or to extend the choice of license to
+ * its licensees as provided above.  However, if you add GPL Version 2 code
+ * and therefore, elected the GPL Version 2 license, then the option applies
+ * only if the new code is made subject to such option by the copyright
+ * holder.
+ */
+package fish.payara.nucleus.config;
+
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.BiFunction;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import javax.inject.Inject;
+
+import org.glassfish.api.StartupRunLevel;
+import org.glassfish.hk2.runlevel.RunLevel;
+import org.jvnet.hk2.annotations.Service;
+
+import com.hazelcast.core.HazelcastInstance;
+import com.hazelcast.core.Member;
+import com.hazelcast.core.MembershipAdapter;
+import com.hazelcast.core.MembershipEvent;
+import com.hazelcast.core.ReplicatedMap;
+
+import fish.payara.nucleus.hazelcast.HazelcastCore;
+
+/**
+ * The {@link ClusteredConfig} can hold configurations that should have a common values that is based on local
+ * configuration values but at the same time accessible to all instances in the cluster.
+ * 
+ * Such cases are not supported by the configuration originating from {@code domain.xml}. First of all non DAS instances
+ * only have their own data available. Secondly with micro instances involved multiple instances can see themselves as
+ * the DAS instance each with its own local value.
+ * 
+ * This configurations allows to take a local configuration property and share it with the other instances. The
+ * effective value for each instance will be computed from a merge {@link BiFunction} on the basis of the local values
+ * shared by the other instances.
+ * 
+ * @author Jan Bernitt
+ * @since 5.201
+ */
+@Service
+@RunLevel(StartupRunLevel.VAL)
+public class ClusteredConfig extends MembershipAdapter {
+
+    /**
+     * Map name prefix used for cluster configuration as accessed using
+     * {@link #getSharedConfiguration(String, Number, BiFunction)}.
+     */
+    private static final String CONFIGURATION_PREFIX = "fish.payara.config:";
+
+    @Inject
+    private HazelcastCore hzCore;
+
+    private String membershipListenerRegistrationId;
+
+    /**
+     * The names of the {@link ReplicatedMap}s holding the instances values of shared configurations.
+     * 
+     * This uses {@link ReplicatedMap}s as responsiveness is important and eventual consistency good enough.
+     */
+    private final Set<String> sharedConfigurations = ConcurrentHashMap.newKeySet();
+
+    @PostConstruct
+    public void postConstruct() {
+        HazelcastInstance hzInstance = hzCore.getInstance();
+        if (hzInstance != null) {
+            membershipListenerRegistrationId = hzInstance.getCluster().addMembershipListener(this);
+        }
+    }
+
+    @PreDestroy
+    public void preDestroy() {
+        HazelcastInstance hzInstance = hzCore.getInstance();
+        if (hzInstance != null && membershipListenerRegistrationId != null) { // can already be null when HZ was shutdown before
+            hzInstance.getCluster().removeMembershipListener(membershipListenerRegistrationId);
+        }
+    }
+
+    /**
+     * Accesses and merges a shared configuration property.
+     * 
+     * Shared configurations are values that use a common value for each instance. The value used is computed by a merge
+     * function from all local values when those become relevant. For example a local value of a disabled feature for
+     * that instance is not relevant and therefore not considered for the shared value.
+     * 
+     * In practice this means when an instance is accessing a value that is a common or shared configuration it calls
+     * this method with its local configuration value which makes it effective for this and other instances.
+     * 
+     * @param name       the globally unique name for the configuration property to read
+     * @param localValue the value as configured locally or a fallback or default value
+     * @param merge      the function to use to resolve both local and shared value being present. The resolved value is
+     *                   always going to be the new shared value. If no value was shared so far the local value is
+     *                   always the new shared and result value.
+     * @return the merged value, local value is the default in case no other instances shared the configuration
+     */
+    public <T extends Number> T getSharedConfiguration(String name, T localValue, BiFunction<T, T, T> merge) {
+        HazelcastInstance hzInstance = hzCore.getInstance();
+        if (hzInstance == null) { // can be null during shutdown
+            return localValue;
+        }
+        String instance = instanceName(hzInstance.getCluster().getLocalMember());
+        String mapName = CONFIGURATION_PREFIX + name;
+        sharedConfigurations.add(mapName);
+        ReplicatedMap<String, T> map = hzInstance.getReplicatedMap(mapName);
+        map.put(instance, localValue);
+        T res = localValue;
+        for (T e : map.values()) {
+            res = merge.apply(res, e);
+        }
+        return res;
+    }
+
+    /**
+     * Can be used to clear the shared value of this instance before the instance is shut down.
+     * 
+     * @param name the globally unique name for the configuration property to clear
+     */
+    public void clearSharedConfiguration(String name) {
+        HazelcastInstance hzInstance = hzCore.getInstance();
+        if (hzInstance != null) { // can be null during shutdown
+            String instance = instanceName(hzInstance.getCluster().getLocalMember());
+            String mapName = CONFIGURATION_PREFIX + name;
+            hzInstance.getReplicatedMap(mapName).remove(instance);
+        }
+    }
+
+    @Override
+    public void memberRemoved(MembershipEvent event) {
+        // remove configuration values from that instance so they no longer have an effect
+        String instance = instanceName(event.getMember());
+        for (String mapName : sharedConfigurations) {
+            hzCore.getInstance().getReplicatedMap(mapName).remove(instance);
+        }
+    }
+
+    private static String instanceName(Member member) {
+        return member.getStringAttribute(HazelcastCore.INSTANCE_ATTRIBUTE);
+    }
+}

nucleus/payara-modules/requesttracing-core/src/main/java/fish/payara/nucleus/requesttracing/RequestTracingService.java

@@ -1,7 +1,7 @@
 /*
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  *
- * Copyright (c) 2016-2019 Payara Foundation and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016-2020 Payara Foundation and/or its affiliates. All rights reserved.
  *
  * The contents of this file are subject to the terms of either the GNU
  * General Public License Version 2 only ("GPL") or the Common Development
@@ -43,6 +43,7 @@
 import com.sun.enterprise.config.serverbeans.Domain;
 import com.sun.enterprise.config.serverbeans.Server;
 import fish.payara.notification.requesttracing.RequestTrace;
+import fish.payara.nucleus.config.ClusteredConfig;
 import fish.payara.nucleus.eventbus.ClusterMessage;
 import fish.payara.nucleus.eventbus.EventBus;
 import fish.payara.nucleus.events.HazelcastEvents;
@@ -95,6 +96,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.TimeUnit;
+import java.util.function.IntSupplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -157,10 +159,13 @@ public class RequestTracingService implements EventListener, ConfigListener, Mon
 
     @Inject
     private NotifierExecutionOptionsFactoryStore executionOptionsFactoryStore;
-    
+
     @Inject
     private HazelcastCore hazelcast;
 
+    @Inject
+    private ClusteredConfig clusteredConfig;
+
     @Inject
     private PayaraExecutorService payaraExecutorService;
 
@@ -265,8 +270,9 @@ public void bootstrapRequestTracingService() {
             // Set up the historic request trace store if enabled
             if (executionOptions.isHistoricTraceStoreEnabled()) {
                 historicRequestTraceStore = RequestTraceStoreFactory.getStore(events, executionOptions.getReservoirSamplingEnabled(), true);
-                historicRequestTraceStore.setSize(executionOptions.getHistoricTraceStoreSize());
-
+                initStoreSize(historicRequestTraceStore, executionOptions::getHistoricTraceStoreSize, "historicRequestTraceStoreSize");
+
+
                 // Disable cleanup task if it's null, less than 0, or reservoir sampling is enabled
                 if (executionOptions.getTraceStoreTimeout() != null 
                         && executionOptions.getTraceStoreTimeout() > 0 
@@ -280,10 +286,10 @@ public void bootstrapRequestTracingService() {
                             0, period, TimeUnit.SECONDS);
                 }
             }
-            
+
             // Set up the general request trace store
             requestTraceStore = RequestTraceStoreFactory.getStore(events, executionOptions.getReservoirSamplingEnabled(), false);
-            requestTraceStore.setSize(executionOptions.getTraceStoreSize());
+            initStoreSize(requestTraceStore, executionOptions::getTraceStoreSize, "requestTraceStoreSize");
 
             // Disable cleanup task if it's null, less than 0, or reservoir sampling is enabled
             if (executionOptions.getTraceStoreTimeout() != null && executionOptions.getTraceStoreTimeout() > 0 
@@ -298,6 +304,18 @@ public void bootstrapRequestTracingService() {
             }
 
             logger.log(Level.INFO, "Payara Request Tracing Service Started with configuration: {0}", executionOptions);
+        } else {
+            clusteredConfig.clearSharedConfiguration("requestTraceStoreSize");
+            clusteredConfig.clearSharedConfiguration("historicRequestTraceStore");
+        }
+    }
+
+    private void initStoreSize(RequestTraceStoreInterface store, IntSupplier size, String clusteredConfigProperty) {
+        if (store.isShared()) {
+            store.setSize(() -> clusteredConfig.getSharedConfiguration(clusteredConfigProperty, size.getAsInt(), Integer::max));
+        } else {
+            clusteredConfig.clearSharedConfiguration(clusteredConfigProperty);
+            store.setSize(size);
         }
     }
 

nucleus/payara-modules/requesttracing-core/src/main/java/fish/payara/nucleus/requesttracing/admin/LocalStrings.properties

@@ -50,7 +50,7 @@ requesttracing.configure.thresholdunit.success=Request Tracing Service Threshold
 requesttracing.configure.sampleratefirst.success=Request Tracing Service Sample Rate First Enabled Value is set to {0}.
 
 requesttracing.configure.store.size.success=Request Tracing Store Size is set to {0}.
-requesttracing.configure.store.size.warning=Please note that from 5.194 onwards the store size refers to the size of a single store shared by all cluster members. Any configuration change regarding the store should be applied to all configurations equally to prevent unbalanced sharing. All affected configurations will be extracted and moved to a central configuration in a future release.
+requesttracing.configure.store.size.warning=Please note that from 5.201 onwards the store size refers to the size of a local store. Should a shared store be used its effective size is the maximum local size of all enabled configurations.
 requesttracing.configure.store.timeout.success=Request Tracing Store Timeout is set to {0}.
 requesttracing.configure.reservoirsampling.enabled.success=Request Tracing Service Reservoir Sampling Enabled Value is set to {0}.
 

nucleus/payara-modules/requesttracing-core/src/main/java/fish/payara/nucleus/requesttracing/store/ClusteredRequestTraceStore.java

@@ -1,7 +1,7 @@
 /*
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  *
- * Copyright (c) 2017-2019 Payara Foundation and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017-2020 Payara Foundation and/or its affiliates. All rights reserved.
  *
  * The contents of this file are subject to the terms of either the GNU
  * General Public License Version 2 only ("GPL") or the Common Development
@@ -46,6 +46,7 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.UUID;
+import java.util.function.IntSupplier;
 import java.util.stream.Collectors;
 
 /**
@@ -55,13 +56,13 @@
 public class ClusteredRequestTraceStore implements RequestTraceStoreInterface, Serializable {
 
     private final Map<UUID, RequestTrace> store;
-    private int maxStoreSize;
+    private IntSupplier maxStoreSize;
 
     private final TraceStorageStrategy strategy;
 
     ClusteredRequestTraceStore(Map<UUID, RequestTrace> store, TraceStorageStrategy strategy) {
         this.store = store;
-        this.maxStoreSize = store.size();
+        this.maxStoreSize = () -> store.size();
         this.strategy = strategy;
     }
 
@@ -73,7 +74,7 @@ public RequestTrace addTrace(RequestTrace trace) {
     @Override
     public RequestTrace addTrace(RequestTrace trace, RequestTrace traceToRemove) {
         store.put(trace.getTraceId(), trace);
-        traceToRemove = strategy.getTraceForRemoval(getTraces(), maxStoreSize, traceToRemove);
+        traceToRemove = strategy.getTraceForRemoval(getTraces(), maxStoreSize.getAsInt(), traceToRemove);
         if (traceToRemove == null) {
             return null;
         }
@@ -92,16 +93,17 @@ public Collection<RequestTrace> getTraces(int limit) {
     }
 
     @Override
-    public void setSize(int maxSize) {
-        while (store.size() > maxSize) {
-            store.remove(strategy.getTraceForRemoval(getTraces(), maxSize, null).getTraceId());
+    public void setSize(IntSupplier maxSize) {
+        int currentMaxSize = maxSize.getAsInt();
+        while (store.size() > currentMaxSize) {
+            store.remove(strategy.getTraceForRemoval(getTraces(), currentMaxSize, null).getTraceId());
         }
         this.maxStoreSize = maxSize;
     }
 
     @Override
     public int getStoreSize() {
-        return this.maxStoreSize;
+        return maxStoreSize.getAsInt();
     }
 
     @Override
@@ -111,4 +113,8 @@ public Collection<RequestTrace> emptyStore() {
         return traces;
     }
 
+    @Override
+    public boolean isShared() {
+        return true;
+    }
 }