[Spark] Fix check to enumerate job IDs

Add a conditional for the new code path (as it's for older versions of
Spark, and causes an additional HTTP request that's not necessary on
newer versions)

checks.d/spark.py

@@ -94,6 +94,9 @@
 SPARK_STANDALONE_MODE = 'spark_standalone_mode'
 SPARK_MESOS_MODE = 'spark_mesos_mode'
 
+# option enabling compatibility mode for Spark ver < 2
+SPARK_PRE_20_MODE = 'spark_pre_20_mode'
+
 # Service Checks
 SPARK_STANDALONE_SERVICE_CHECK = 'spark.standalone_master.can_connect'
 YARN_SERVICE_CHECK = 'spark.resource_manager.can_connect'
@@ -198,6 +201,7 @@ def __init__(self, name, init_config, agentConfig, instances=None):
         AgentCheck.__init__(self, name, init_config, agentConfig, instances)
         self.previous_jobs = {}
         self.previous_stages = {}
+        self._spark_pre_20_mode = False
 
     def check(self, instance):
         # Get additional tags from the conf file
@@ -263,6 +267,11 @@ def _get_running_apps(self, instance, tags):
             cluster_mode = SPARK_YARN_MODE
 
         if cluster_mode == SPARK_STANDALONE_MODE:
+            # check for PRE-20
+            pre20 = instance.get(SPARK_PRE_20_MODE)
+            if pre20 is not None:
+                if pre20 is True:
+                    self._spark_pre_20_mode = True
             return self._standalone_init(master_address)
 
         elif cluster_mode == SPARK_MESOS_MODE:
@@ -294,17 +303,32 @@ def _standalone_init(self, spark_master_address):
                 app_name = app.get('name')
 
                 # Parse through the HTML to grab the application driver's link
-                app_url = self._get_standalone_app_url(app_id, spark_master_address)
-
-                if app_id and app_name and app_url:
-                    running_apps[app_id] = (app_name, app_url)
+                try:
+                    app_url = self._get_standalone_app_url(app_id, spark_master_address)
+
+                    if app_id and app_name and app_url:
+                        if self._spark_pre_20_mode:
+                            self.log.debug('Getting application list in pre-20 mode')
+                            applist = self._rest_request_to_json(app_url,
+                                SPARK_APPS_PATH,
+                                SPARK_STANDALONE_SERVICE_CHECK)
+                            for appl in applist:
+                                aid = appl.get('id')
+                                aname = appl.get('name')
+                                running_apps[aid] = (aname, app_url)
+                        else:
+                            running_apps[app_id] = (app_name, app_url)
+                except:
+                    # it's possible for the requests to fail if the job
+                    # completed since we got the list of apps.  Just continue
+                    pass
 
         # Report success after gathering metrics from Spark master
         self.service_check(SPARK_STANDALONE_SERVICE_CHECK,
             AgentCheck.OK,
             tags=['url:%s' % spark_master_address],
             message='Connection to Spark master "%s" was successful' % spark_master_address)
-
+        self.log.info("Returning running apps %s" % running_apps)
         return running_apps
 
     def _mesos_init(self, master_address):
@@ -624,6 +648,7 @@ def _rest_request(self, address, object_path, service_name, *args, **kwargs):
             url = urljoin(url, '?' + query)
 
         try:
+            self.log.debug('Spark check URL: %s' % url)
             response = requests.get(url)
             response.raise_for_status()
 

conf.d/spark.yaml.example

@@ -28,6 +28,10 @@ instances:
     # spark_cluster_mode: spark_standalone_mode
     # spark_cluster_mode: spark_mesos_mode
 
+    # To use an older (versions prior to 2.0) Standalone Spark cluster, 
+    # the 'spark_pre_20_mode' must be set
+    # spark_pre_20_mode: true
+
     # A Required friendly name for the cluster.
     # cluster_name: MySparkCluster