SOLR-17381: Make CLUSTERSTATUS configurable (#2599)

Make CLUSTERSTATUS configurable to improve performance by allowing retrieval of specific information with these boolean parameters: liveNodes, clusterProperties, aliases, roles, and includeAll Updated SolrJ HttpClusterStateProvider (state from Solr nodes) to use these parameters so that only required information is fetched and returned.
apache · Aug 9, 2024 · c7af8b8 · c7af8b8
1 parent 5abeb4b
commit c7af8b8
Show file tree

Hide file tree

Showing 8 changed files with 219 additions and 113 deletions.
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
@@ -118,6 +118,9 @@ Optimizations
 * SOLR-17102: The VersionBucket indexing lock mechanism was replaced with something just as fast yet
   that which consumes almost no memory, saving 1MB of memory per SolrCore.  (David Smiley)
 
+* SOLR-17381: Make CLUSTERSTATUS request configurable to improve performance by allowing retrieval of specific information,
+  reducing unnecessary data fetching. (Aparna Suresh, David Smiley)
+
 * SOLR-17396: Reduce thread contention in ZkStateReader.getCollectionProperties(). (Aparna Suresh, David Smiley, Paul McArthur)
 
 Bug Fixes

diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
@@ -34,19 +34,24 @@
 import org.apache.solr.common.cloud.PerReplicaStates;
 import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
 import org.apache.solr.common.util.Utils;
 import org.apache.zookeeper.KeeperException;
 
 public class ClusterStatus {
   private final ZkStateReader zkStateReader;
-  private final ZkNodeProps message;
+  private final SolrParams solrParams;
   private final String collection; // maybe null
 
+  public static final String INCLUDE_ALL = "includeAll";
+  public static final String LIVENODES_PROP = "liveNodes";
+  public static final String CLUSTER_PROP = "clusterProperties";
+  public static final String ALIASES_PROP = "aliases";
+
   /** Shard / collection health state. */
   public enum Health {
     /** All replicas up, leader exists. */
@@ -89,16 +94,72 @@ public static Health combine(Collection<Health> states) {
     }
   }
 
-  public ClusterStatus(ZkStateReader zkStateReader, ZkNodeProps props) {
+  public ClusterStatus(ZkStateReader zkStateReader, SolrParams params) {
     this.zkStateReader = zkStateReader;
-    this.message = props;
-    collection = props.getStr(ZkStateReader.COLLECTION_PROP);
+    this.solrParams = params;
+    collection = params.get(ZkStateReader.COLLECTION_PROP);
   }
 
   public void getClusterStatus(NamedList<Object> results)
       throws KeeperException, InterruptedException {
+    NamedList<Object> clusterStatus = new SimpleOrderedMap<>();
+
+    boolean includeAll = solrParams.getBool(INCLUDE_ALL, true);
+    boolean withLiveNodes = solrParams.getBool(LIVENODES_PROP, includeAll);
+    boolean withClusterProperties = solrParams.getBool(CLUSTER_PROP, includeAll);
+    boolean withRoles = solrParams.getBool(ZkStateReader.ROLES_PROP, includeAll);
+    boolean withCollection = includeAll || (collection != null);
+    boolean withAliases = solrParams.getBool(ALIASES_PROP, includeAll);
+
+    List<String> liveNodes = null;
+    if (withLiveNodes || collection != null) {
+      liveNodes =
+          zkStateReader.getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true);
+      // add live_nodes
+      if (withLiveNodes) clusterStatus.add("live_nodes", liveNodes);
+    }
+
+    Aliases aliases = null;
+    if (withCollection || withAliases) {
+      aliases = zkStateReader.getAliases();
+    }
+
+    if (withCollection) {
+      assert liveNodes != null;
+      fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases);
+    }
+
+    if (withAliases) {
+      addAliasMap(aliases, clusterStatus);
+    }
+
+    if (withClusterProperties) {
+      Map<String, Object> clusterProps = zkStateReader.getClusterProperties();
+      if (clusterProps == null) {
+        clusterProps = Collections.emptyMap();
+      }
+      clusterStatus.add("properties", clusterProps);
+    }
+
+    // add the roles map
+    if (withRoles) {
+      Map<?, ?> roles = Collections.emptyMap();
+      if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) {
+        roles =
+            (Map<?, ?>)
+                Utils.fromJSON(
+                    zkStateReader.getZkClient().getData(ZkStateReader.ROLES, null, null, true));
+      }
+      clusterStatus.add("roles", roles);
+    }
+
+    results.add("cluster", clusterStatus);
+  }
+
+  private void fetchClusterStatusForCollOrAlias(
+      NamedList<Object> clusterStatus, List<String> liveNodes, Aliases aliases) {
+
     // read aliases
-    Aliases aliases = zkStateReader.getAliases();
     Map<String, List<String>> collectionVsAliases = new HashMap<>();
     Map<String, List<String>> aliasVsCollections = aliases.getCollectionAliasListMap();
     for (Map.Entry<String, List<String>> entry : aliasVsCollections.entrySet()) {
@@ -112,18 +173,10 @@ public void getClusterStatus(NamedList<Object> results)
       }
     }
 
-    Map<?, ?> roles = null;
-    if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) {
-      roles =
-          (Map<?, ?>)
-              Utils.fromJSON(
-                  zkStateReader.getZkClient().getData(ZkStateReader.ROLES, null, null, true));
-    }
-
     ClusterState clusterState = zkStateReader.getClusterState();
 
-    String routeKey = message.getStr(ShardParams._ROUTE_);
-    String shard = message.getStr(ZkStateReader.SHARD_ID_PROP);
+    String routeKey = solrParams.get(ShardParams._ROUTE_);
+    String shard = solrParams.get(ZkStateReader.SHARD_ID_PROP);
 
     Map<String, DocCollection> collectionsMap = null;
     if (collection == null) {
@@ -139,6 +192,7 @@ public void getClusterStatus(NamedList<Object> results)
     if (didNotFindCollection && isAlias) {
       // In this case this.collection is an alias name not a collection
       // get all collections and filter out collections not in the alias
+      // clusterState.getCollectionsMap() should be replaced with an inexpensive call
       collectionsMap =
           clusterState.getCollectionsMap().entrySet().stream()
               .filter((entry) -> aliasVsCollections.get(collection).contains(entry.getKey()))
@@ -191,43 +245,25 @@ public void getClusterStatus(NamedList<Object> results)
       }
       String configName = clusterStateCollection.getConfigName();
       collectionStatus.put("configName", configName);
-      if (message.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) {
+      if (solrParams.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) {
         PerReplicaStates prs = clusterStateCollection.getPerReplicaStates();
         collectionStatus.put("PRS", prs);
       }
       collectionProps.add(name, collectionStatus);
     }
 
-    List<String> liveNodes =
-        zkStateReader.getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true);
-
     // now we need to walk the collectionProps tree to cross-check replica state with live nodes
     crossCheckReplicaStateWithLiveNodes(liveNodes, collectionProps);
 
-    NamedList<Object> clusterStatus = new SimpleOrderedMap<>();
     clusterStatus.add("collections", collectionProps);
+  }
 
-    // read cluster properties
-    Map<String, Object> clusterProps = zkStateReader.getClusterProperties();
-    if (clusterProps != null && !clusterProps.isEmpty()) {
-      clusterStatus.add("properties", clusterProps);
-    }
-
+  private void addAliasMap(Aliases aliases, NamedList<Object> clusterStatus) {
     // add the alias map too
     Map<String, String> collectionAliasMap = aliases.getCollectionAliasMap(); // comma delim
     if (!collectionAliasMap.isEmpty()) {
       clusterStatus.add("aliases", collectionAliasMap);
     }
-
-    // add the roles map
-    if (roles != null) {
-      clusterStatus.add("roles", roles);
-    }
-
-    // add live_nodes
-    clusterStatus.add("live_nodes", liveNodes);
-
-    results.add("cluster", clusterStatus);
   }
 
   /**

diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -100,7 +100,6 @@
 import static org.apache.solr.common.params.CommonParams.VALUE_LONG;
 import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION;
 import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY;
-import static org.apache.solr.common.params.ShardParams._ROUTE_;
 import static org.apache.solr.common.util.StrUtils.formatString;
 
 import java.lang.invoke.MethodHandles;
@@ -975,10 +974,7 @@ public Map<String, Object> execute(
     CLUSTERSTATUS_OP(
         CLUSTERSTATUS,
         (req, rsp, h) -> {
-          Map<String, Object> all =
-              copy(req.getParams(), null, COLLECTION_PROP, SHARD_ID_PROP, _ROUTE_, "prs");
-          new ClusterStatus(
-                  h.coreContainer.getZkController().getZkStateReader(), new ZkNodeProps(all))
+          new ClusterStatus(h.coreContainer.getZkController().getZkStateReader(), req.getParams())
               .getClusterStatus(rsp.getValues());
           return null;
         }),

diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc
@@ -99,6 +99,51 @@ Multiple shard names can be specified as a comma-separated list.
 +
 This can be used if you need the details of the shard where a particular document belongs to and you don't know which shard it falls under.
 
+`aliases`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: will default to the default value of `includeAll` parameter specified below
+|===
++
+
+`liveNodes`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: will default to the default value of `includeAll` parameter specified below
+|===
++
+If set to true, returns the status of live nodes in the cluster.
+
+`clusterProperties`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: will default to the default value of `includeAll` parameter specified below
+|===
++
+If set to true, returns the properties of the cluster.
+
+`roles`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: will default to the default value of `includeAll` parameter specified below
+|===
++
+If set to true, returns the roles within the cluster.
+
+`includeAll`::
++
+[%autowidth,frame=none]
+|===
+|Optional |Default: true
+|===
++
+If set to `true`, returns all information pertaining to live nodes, collections, aliases, cluster properties, roles, etc.
+If set to `false`, the information returned is based on the other specified parameters.
+
 === CLUSTERSTATUS Response
 
 The response will include the status of the request and the status of the cluster.