[pantsd] Launch the daemon via the thin client. (#4931)

Problem

Currently, the daemon is launched in the middle of a pants run using a Subsystem. Because this happens in the middle of the run and relies on things like Subsystem and options initialization the first run is always "throw away" and doesn't use the daemon. Furthermore, having the lifecycle in the middle of the run vs at the beginning hobbles our ability to cleanly perform lifecycle operations like inline daemon restarts.

Solution

Move all daemon related options to bootstrap options to enable use of the previous daemon-supporting Subsystem instances earlier in the run. Move the responsibility of launching the daemon to the thin client runner, so that it can synchronously start and use the daemon inline.

Result

The thin client can now launch the daemon itself and use it for the first run.

src/python/pants/backend/jvm/tasks/nailgun_task.py

@@ -9,11 +9,11 @@
 
 from pants.backend.jvm.tasks.jvm_tool_task_mixin import JvmToolTaskMixin
 from pants.base.exceptions import TaskError
+from pants.init.subprocess import Subprocess
 from pants.java import util
 from pants.java.executor import SubprocessExecutor
 from pants.java.jar.jar_dependency import JarDependency
 from pants.java.nailgun_executor import NailgunExecutor, NailgunProcessGroup
-from pants.pantsd.subsystem.subprocess import Subprocess
 from pants.task.task import Task, TaskBase
 
 

src/python/pants/bin/BUILD

@@ -34,10 +34,12 @@ python_library(
     'src/python/pants/init',
     'src/python/pants/option',
     'src/python/pants/reporting',
+    'src/python/pants/pantsd:pants_daemon_launcher',
     'src/python/pants/scm/subsystems:changed',
     'src/python/pants/subsystem',
     'src/python/pants/task',
     'src/python/pants/util:contextutil',
+    'src/python/pants/util:collections',
     'src/python/pants/util:dirutil',
     'src/python/pants/util:filtering',
     'src/python/pants/util:memo',

src/python/pants/bin/goal_runner.py

@@ -23,7 +23,7 @@
 from pants.goal.goal import Goal
 from pants.goal.run_tracker import RunTracker
 from pants.help.help_printer import HelpPrinter
-from pants.init.pants_daemon_launcher import PantsDaemonLauncher
+from pants.init.subprocess import Subprocess
 from pants.init.target_roots import TargetRoots
 from pants.java.nailgun_executor import NailgunProcessGroup
 from pants.reporting.reporting import Reporting
@@ -152,14 +152,7 @@ def generate_targets(specs):
 
     return list(generate_targets(specs))
 
-  def _maybe_launch_pantsd(self, pantsd_launcher):
-    """Launches pantsd if configured to do so."""
-    if self._global_options.enable_pantsd:
-      # Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on.
-      with self._run_tracker.new_workunit(name='pantsd', labels=[WorkUnitLabel.SETUP]):
-        pantsd_launcher.maybe_launch()
-
-  def _setup_context(self, pantsd_launcher):
+  def _setup_context(self):
     with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]):
       self._build_graph, self._address_mapper, spec_roots = self._init_graph(
         self._global_options.enable_v2_engine,
@@ -189,15 +182,12 @@ def _setup_context(self, pantsd_launcher):
                         build_graph=self._build_graph,
                         build_file_parser=self._build_file_parser,
                         address_mapper=self._address_mapper,
-                        invalidation_report=invalidation_report,
-                        pantsd_launcher=pantsd_launcher)
+                        invalidation_report=invalidation_report)
       return goals, context
 
   def setup(self):
-    pantsd_launcher = PantsDaemonLauncher.Factory.global_instance().create(EngineInitializer)
-    self._maybe_launch_pantsd(pantsd_launcher)
     self._handle_help(self._help_request)
-    goals, context = self._setup_context(pantsd_launcher)
+    goals, context = self._setup_context()
     return GoalRunner(context=context,
                       goals=goals,
                       run_tracker=self._run_tracker,
@@ -234,7 +224,7 @@ def subsystems(cls):
       RunTracker,
       Changed.Factory,
       BinaryUtil.Factory,
-      PantsDaemonLauncher.Factory,
+      Subprocess.Factory
     }
 
   def _execute_engine(self):

src/python/pants/bin/pants_runner.py

@@ -29,29 +29,24 @@ def __init__(self, exiter, args=None, env=None):
     self._args = args or sys.argv
     self._env = env or os.environ
 
-  def _run(self, is_remote, exiter, args, env, process_metadata_dir=None, options_bootstrapper=None):
-    if is_remote:
+  def run(self):
+    options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
+    bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
+
+    if bootstrap_options.enable_pantsd:
       try:
-        return RemotePantsRunner(exiter, args, env, process_metadata_dir).run()
-      except RemotePantsRunner.RECOVERABLE_EXCEPTIONS as e:
-        # N.B. RemotePantsRunner will raise one of RECOVERABLE_EXCEPTIONS in the event we
-        # encounter a failure while discovering or initially connecting to the pailgun. In
-        # this case, we fall back to LocalPantsRunner which seamlessly executes the requested
-        # run and bootstraps pantsd for use in subsequent runs.
-        logger.debug('caught client exception: {!r}, falling back to LocalPantsRunner'.format(e))
+        return RemotePantsRunner(self._exiter,
+                                 self._args,
+                                 self._env,
+                                 bootstrap_options.pants_subprocessdir,
+                                 bootstrap_options).run()
+      except RemotePantsRunner.Fallback as e:
+        logger.debug('caught client exception: {!r}, falling back to non-daemon mode'.format(e))
 
     # N.B. Inlining this import speeds up the python thin client run by about 100ms.
     from pants.bin.local_pants_runner import LocalPantsRunner
 
-    return LocalPantsRunner(exiter, args, env, options_bootstrapper=options_bootstrapper).run()
-
-  def run(self):
-    options_bootstrapper = OptionsBootstrapper(env=self._env, args=self._args)
-    global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
-
-    return self._run(is_remote=global_bootstrap_options.enable_pantsd,
-                     exiter=self._exiter,
-                     args=self._args,
-                     env=self._env,
-                     process_metadata_dir=global_bootstrap_options.pants_subprocessdir,
-                     options_bootstrapper=options_bootstrapper)
+    return LocalPantsRunner(self._exiter,
+                            self._args,
+                            self._env,
+                            options_bootstrapper=options_bootstrapper).run()

src/python/pants/bin/remote_pants_runner.py

@@ -5,30 +5,41 @@
 from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                         unicode_literals, with_statement)
 
+import logging
 import signal
 import sys
 from contextlib import contextmanager
 
+from pants.bin.engine_initializer import EngineInitializer
 from pants.java.nailgun_client import NailgunClient
 from pants.java.nailgun_protocol import NailgunProtocol
-from pants.pantsd.process_manager import ProcessMetadataManager
+from pants.pantsd.pants_daemon_launcher import PantsDaemonLauncher
+from pants.util.collections import combined_dict
+
+
+logger = logging.getLogger(__name__)
 
 
 class RemotePantsRunner(object):
   """A thin client variant of PantsRunner."""
 
-  class PortNotFound(Exception): pass
+  class Fallback(Exception):
+    """Raised when fallback to an alternate execution mode is requested."""
+
+  class PortNotFound(Exception):
+    """Raised when the pailgun port can't be found."""
 
   PANTS_COMMAND = 'pants'
   RECOVERABLE_EXCEPTIONS = (PortNotFound, NailgunClient.NailgunConnectionError)
 
-  def __init__(self, exiter, args, env, process_metadata_dir=None,
-               stdin=None, stdout=None, stderr=None):
+  def __init__(self, exiter, args, env, process_metadata_dir,
+               bootstrap_options, stdin=None, stdout=None, stderr=None):
     """
     :param Exiter exiter: The Exiter instance to use for this run.
     :param list args: The arguments (e.g. sys.argv) for this run.
     :param dict env: The environment (e.g. os.environ) for this run.
     :param str process_metadata_dir: The directory in which process metadata is kept.
+    :param Options bootstrap_options: The Options bag containing the bootstrap options.
     :param file stdin: The stream representing stdin.
     :param file stdout: The stream representing stdout.
     :param file stderr: The stream representing stderr.
@@ -37,17 +48,11 @@ def __init__(self, exiter, args, env, process_metadata_dir=None,
     self._args = args
     self._env = env
     self._process_metadata_dir = process_metadata_dir
+    self._bootstrap_options = bootstrap_options
     self._stdin = stdin or sys.stdin
     self._stdout = stdout or sys.stdout
     self._stderr = stderr or sys.stderr
-    self._port = self._retrieve_pailgun_port()
-    if not self._port:
-      raise self.PortNotFound('unable to locate pailgun port!')
-
-  @staticmethod
-  def _combine_dicts(*dicts):
-    """Combine one or more dicts into a new, unified dict (dicts to the right take precedence)."""
-    return {k: v for d in dicts for k, v in d.items()}
+    self._launcher = PantsDaemonLauncher(self._bootstrap_options, EngineInitializer)
 
   @contextmanager
   def _trapped_control_c(self, client):
@@ -62,17 +67,36 @@ def handle_control_c(signum, frame):
     finally:
       signal.signal(signal.SIGINT, existing_sigint_handler)
 
-  def _retrieve_pailgun_port(self):
-    return ProcessMetadataManager(
-      self._process_metadata_dir).read_metadata_by_name('pantsd', 'socket_pailgun', int)
+  def _setup_logging(self):
+    """Sets up basic stdio logging for the thin client."""
+    log_level = logging.getLevelName(self._bootstrap_options.level.upper())
 
-  def run(self, args=None):
+    formatter = logging.Formatter('%(levelname)s] %(message)s')
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setLevel(log_level)
+    handler.setFormatter(formatter)
+
+    root = logging.getLogger()
+    root.setLevel(log_level)
+    root.addHandler(handler)
+
+  def _find_or_launch_pantsd(self):
+    """Launches pantsd if configured to do so.
+
+    :returns: The port pantsd can be found on.
+    :rtype: int
+    """
+    return self._launcher.maybe_launch()
+
+  def _connect_and_execute(self, port):
     # Merge the nailgun TTY capability environment variables with the passed environment dict.
     ng_env = NailgunProtocol.isatty_to_env(self._stdin, self._stdout, self._stderr)
-    modified_env = self._combine_dicts(self._env, ng_env)
+    modified_env = combined_dict(self._env, ng_env)
+
+    assert isinstance(port, int), 'port {} is not an integer!'.format(port)
 
     # Instantiate a NailgunClient.
-    client = NailgunClient(port=self._port,
+    client = NailgunClient(port=port,
                            ins=self._stdin,
                            out=self._stdout,
                            err=self._stderr,
@@ -84,3 +108,13 @@ def run(self, args=None):
 
     # Exit.
     self._exiter.exit(result)
+
+  def run(self, args=None):
+    self._setup_logging()
+    port = self._find_or_launch_pantsd()
+
+    logger.debug('connecting to pailgun on port {}'.format(port))
+    try:
+      self._connect_and_execute(port)
+    except self.RECOVERABLE_EXCEPTIONS as e:
+      raise self.Fallback(e)

src/python/pants/core_tasks/pantsd_kill.py

@@ -6,6 +6,7 @@
                         unicode_literals, with_statement)
 
 from pants.base.exceptions import TaskError
+from pants.pantsd.pants_daemon_launcher import PantsDaemonLauncher
 from pants.pantsd.process_manager import ProcessManager
 from pants.task.task import Task
 
@@ -15,6 +16,6 @@ class PantsDaemonKill(Task):
 
   def execute(self):
     try:
-      self.context.pantsd_launcher.terminate()
+      PantsDaemonLauncher(self.get_options()).terminate()
     except ProcessManager.NonResponsiveProcess as e:
       raise TaskError('failure while terminating pantsd: {}'.format(e))

src/python/pants/core_tasks/register.py

@@ -59,7 +59,9 @@ def register_goals():
   # Pantsd.
   kill_pantsd = task(name='kill-pantsd', action=PantsDaemonKill)
   kill_pantsd.install()
-  kill_pantsd.install('clean-all')
+  # Kill pantsd/watchman first, so that they're not using any files
+  # in .pants.d at the time of removal.
+  kill_pantsd.install('clean-all', first=True)
 
   # Reporting server.
   # TODO: The reporting server should be subsumed into pantsd, and not run via a task.

src/python/pants/engine/native.py

@@ -561,9 +561,9 @@ class Native(object):
   """Encapsulates fetching a platform specific version of the native portion of the engine."""
 
   @staticmethod
-  def create(options):
+  def create(bootstrap_options):
     """:param options: Any object that provides access to bootstrap option values."""
-    return Native(options.native_engine_visualize_to)
+    return Native(bootstrap_options.native_engine_visualize_to)
 
   def __init__(self, visualize_to_dir):
     """

src/python/pants/goal/context.py

@@ -61,7 +61,7 @@ def fatal(self, *msg_elements):
   def __init__(self, options, run_tracker, target_roots,
                requested_goals=None, target_base=None, build_graph=None,
                build_file_parser=None, address_mapper=None, console_outstream=None, scm=None,
-               workspace=None, invalidation_report=None, pantsd_launcher=None):
+               workspace=None, invalidation_report=None):
     self._options = options
     self.build_graph = build_graph
     self.build_file_parser = build_file_parser
@@ -80,7 +80,6 @@ def __init__(self, options, run_tracker, target_roots,
     self._workspace = workspace or (ScmWorkspace(self._scm) if self._scm else None)
     self._replace_targets(target_roots)
     self._invalidation_report = invalidation_report
-    self._pantsd_launcher = pantsd_launcher
 
   @property
   def options(self):
@@ -151,10 +150,6 @@ def workspace(self):
   def invalidation_report(self):
     return self._invalidation_report
 
-  @property
-  def pantsd_launcher(self):
-    return self._pantsd_launcher
-
   def __str__(self):
     ident = Target.identify(self.targets())
     return 'Context(id:{}, targets:{})'.format(ident, self.targets())

src/python/pants/init/BUILD

@@ -1,7 +1,6 @@
 # Copyright 2017 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-
 python_library(
   dependencies=[
     ':plugins',
@@ -21,12 +20,6 @@ python_library(
     'src/python/pants/goal:run_tracker',
     'src/python/pants/logging',
     'src/python/pants/option',
-    'src/python/pants/pantsd:pants_daemon',
-    'src/python/pants/pantsd/service:fs_event_service',
-    'src/python/pants/pantsd/service:pailgun_service',
-    'src/python/pants/pantsd/service:scheduler_service',
-    'src/python/pants/pantsd/subsystem:subprocess',
-    'src/python/pants/pantsd/subsystem:watchman_launcher',
     'src/python/pants/process',
     'src/python/pants/python',
     'src/python/pants/subsystem',