diff --git a/composer.json b/composer.json index a5a68f8cb9f7..2f6e7ca5eb33 100644 --- a/composer.json +++ b/composer.json @@ -114,8 +114,7 @@ "google-cloud": "dev/google-cloud" }, "bin": [ - "src/Core/bin/google-cloud-batch", - "src/Debugger/bin/google-cloud-debugger" + "src/Core/bin/google-cloud-batch" ], "extra": { "component": { diff --git a/src/Core/Batch/BatchDaemon.php b/src/Core/Batch/BatchDaemon.php index c49952fdbe41..539b5218aa6f 100644 --- a/src/Core/Batch/BatchDaemon.php +++ b/src/Core/Batch/BatchDaemon.php @@ -37,13 +37,11 @@ class BatchDaemon use BatchDaemonTrait; use HandleFailureTrait; use SysvTrait; + use InterruptTrait; /* @var BatchRunner */ private $runner; - /* @var bool */ - private $shutdown; - /* @var array */ private $descriptorSpec; @@ -72,53 +70,31 @@ public function __construct($entrypoint) 1 => ['file', 'php://stdout', 'w'], 2 => ['file', 'php://stderr', 'w'] ]; - // setup signal handlers - pcntl_signal(SIGTERM, [$this, "sigHandler"]); - pcntl_signal(SIGINT, [$this, "sigHandler"]); - pcntl_signal(SIGHUP, [$this, "sigHandler"]); - pcntl_signal(SIGALRM, [$this, "sigHandler"]); + $this->command = sprintf('exec php -d auto_prepend_file="" %s daemon', $entrypoint); $this->initFailureFile(); } - /** - * A signal handler for setting the terminate switch. - * {@see http://php.net/manual/en/function.pcntl-signal.php} - * - * @param int $signo The received signal. - * @param mixed $siginfo [optional] An array representing the signal - * information. **Defaults to** null. - * - * @return void - */ - public function sigHandler($signo, $signinfo = null) - { - switch ($signo) { - case SIGINT: - case SIGTERM: - $this->shutdown = true; - break; - } - } - /** * A loop for the parent. * * @return void */ - public function runParent() + public function run() { + $this->setupSignalHandlers(); + $procs = []; while (true) { $jobs = $this->runner->getJobs(); foreach ($jobs as $job) { - if (! array_key_exists($job->getIdentifier(), $procs)) { - $procs[$job->getIdentifier()] = []; + if (! array_key_exists($job->identifier(), $procs)) { + $procs[$job->identifier()] = []; } - while (count($procs[$job->getIdentifier()]) > $job->getWorkerNum()) { + while (count($procs[$job->identifier()]) > $job->numWorkers()) { // Stopping an excessive child. echo 'Stopping an excessive child.' . PHP_EOL; - $proc = array_pop($procs[$job->getIdentifier()]); + $proc = array_pop($procs[$job->identifier()]); $status = proc_get_status($proc); // Keep sending SIGTERM until the child exits. while ($status['running'] === true) { @@ -128,10 +104,10 @@ public function runParent() } @proc_close($proc); } - for ($i = 0; $i < $job->getWorkerNum(); $i++) { + for ($i = 0; $i < $job->numWorkers(); $i++) { $needStart = false; - if (array_key_exists($i, $procs[$job->getIdentifier()])) { - $status = proc_get_status($procs[$job->getIdentifier()][$i]); + if (array_key_exists($i, $procs[$job->identifier()])) { + $status = proc_get_status($procs[$job->identifier()][$i]); if ($status['running'] !== true) { $needStart = true; } @@ -140,8 +116,8 @@ public function runParent() } if ($needStart) { echo 'Starting a child.' . PHP_EOL; - $procs[$job->getIdentifier()][$i] = proc_open( - sprintf('%s %d', $this->command, $job->getIdNum()), + $procs[$job->identifier()][$i] = proc_open( + sprintf('%s %d', $this->command, $job->id()), $this->descriptorSpec, $pipes ); @@ -173,64 +149,13 @@ public function runParent() } /** - * A loop for the children. + * Fetch the child job by id. * - * @param int $idNum Numeric id for the job. - * @return void + * @param int $idNum The id of the job to find + * @return JobInterface */ - public function runChild($idNum) + public function job($idNum) { - // child process - $sysvKey = $this->getSysvKey($idNum); - $q = msg_get_queue($sysvKey); - $items = []; - $job = $this->runner->getJobFromIdNum($idNum); - $period = $job->getCallPeriod(); - $lastInvoked = microtime(true); - $batchSize = $job->getBatchSize(); - while (true) { - // Fire SIGALRM after 1 second to unblock the blocking call. - pcntl_alarm(1); - if (msg_receive( - $q, - 0, - $type, - 8192, - $message, - true, - 0, // blocking mode - $errorcode - )) { - if ($type === self::$typeDirect) { - $items[] = $message; - } elseif ($type === self::$typeFile) { - $items[] = unserialize(file_get_contents($message)); - @unlink($message); - } - } - pcntl_signal_dispatch(); - // It runs the job when - // 1. Number of items reaches the batchSize. - // 2-a. Count is >0 and the current time is larger than lastInvoked + period. - // 2-b. Count is >0 and the shutdown flag is true. - if ((count($items) >= $batchSize) - || (count($items) > 0 - && (microtime(true) > $lastInvoked + $period - || $this->shutdown))) { - printf( - 'Running the job with %d items' . PHP_EOL, - count($items) - ); - if (! $job->run($items)) { - $this->handleFailure($idNum, $items); - } - $items = []; - $lastInvoked = microtime(true); - } - gc_collect_cycles(); - if ($this->shutdown) { - exit; - } - } + return $this->runner->getJobFromIdNum($idNum); } } diff --git a/src/Core/Batch/BatchDaemonTrait.php b/src/Core/Batch/BatchDaemonTrait.php index b24123116802..30592c3323ae 100644 --- a/src/Core/Batch/BatchDaemonTrait.php +++ b/src/Core/Batch/BatchDaemonTrait.php @@ -27,9 +27,6 @@ */ trait BatchDaemonTrait { - private static $typeDirect = 1; - private static $typeFile = 2; - /** * Returns whether or not the BatchDaemon is running. * diff --git a/src/Core/Batch/BatchJob.php b/src/Core/Batch/BatchJob.php index 01a0ea2d7097..38b3640800ac 100644 --- a/src/Core/Batch/BatchJob.php +++ b/src/Core/Batch/BatchJob.php @@ -17,6 +17,8 @@ namespace Google\Cloud\Core\Batch; +use Google\Cloud\Core\SysvTrait; + /** * Represent batch jobs. * @@ -25,47 +27,34 @@ * incompatible ways. Please use with caution, and test thoroughly when * upgrading. */ -class BatchJob +class BatchJob implements JobInterface { const DEFAULT_BATCH_SIZE = 100; const DEFAULT_CALL_PERIOD = 2.0; const DEFAULT_WORKERS = 1; - /** - * @var string - */ - private $identifier; + use JobTrait; + use SysvTrait; + use InterruptTrait; + use HandleFailureTrait; /** - * @var callable + * @var callable The batch job handler. This callable accepts an array + * of items and should return a boolean. */ private $func; /** - * @var int - */ - private $idNum; - - /** - * @var string - */ - private $bootstrapFile; - - /** - * @var int + * @var int The size of the batch. */ private $batchSize; /** - * @var float + * @var float The period in seconds from the last execution to force + * executing the job. */ private $callPeriod; - /** - * @var int - */ - private $workerNum; - /** * @param string $identifier Unique identifier of the job. * @param callable $func Any Callable except for Closure. The callable @@ -77,7 +66,7 @@ class BatchJob * @type int $batchSize The size of the batch. **Defaults to** `100`. * @type float $callPeriod The period in seconds from the last execution * to force executing the job. **Defaults to** `2.0`. - * @type int $workerNum The number of child processes. It only takes + * @type int $numWorkers The number of child processes. It only takes * effect with the {@see \Google\Cloud\Core\Batch\BatchDaemon}. * **Defaults to** `1`. * @type string $bootstrapFile A file to load before executing the @@ -90,55 +79,100 @@ public function __construct( $idNum, array $options = [] ) { + $options += [ + 'batchSize' => self::DEFAULT_BATCH_SIZE, + 'callPeriod' => self::DEFAULT_CALL_PERIOD, + 'bootstrapFile' => null, + 'numWorkers' => self::DEFAULT_WORKERS + ]; $this->identifier = $identifier; $this->func = $func; - $this->idNum = $idNum; - $this->batchSize = array_key_exists('batchSize', $options) - ? $options['batchSize'] - : self::DEFAULT_BATCH_SIZE; - $this->callPeriod = array_key_exists('callPeriod', $options) - ? $options['callPeriod'] - : self::DEFAULT_CALL_PERIOD; - $this->bootstrapFile = array_key_exists('bootstrapFile', $options) - ? $options['bootstrapFile'] - : null; - $this->workerNum = array_key_exists('workerNum', $options) - ? $options['workerNum'] - : self::DEFAULT_WORKERS; + $this->id = $idNum; + $this->batchSize = $options['batchSize']; + $this->callPeriod = $options['callPeriod']; + $this->bootstrapFile = $options['bootstrapFile']; + $this->numWorkers = $options['numWorkers']; } /** - * Run the job with the given items. - * - * @param array $items An array of items. - * - * @return bool the result of the callback + * Run the job. */ - public function run(array $items) + public function run() { - if (! is_null($this->bootstrapFile)) { + $this->setupSignalHandlers(); + + $sysvKey = $this->getSysvKey($this->id); + $q = msg_get_queue($sysvKey); + $items = []; + $lastInvoked = microtime(true); + + if (!is_null($this->bootstrapFile)) { require_once($this->bootstrapFile); } - return call_user_func_array($this->func, [$items]); - } - /** - * @return int - */ - public function getIdNum() - { - return $this->idNum; + while (true) { + // Fire SIGALRM after 1 second to unblock the blocking call. + pcntl_alarm(1); + if (msg_receive( + $q, + 0, + $type, + 8192, + $message, + true, + 0, // blocking mode + $errorcode + )) { + if ($type === self::$typeDirect) { + $items[] = $message; + } elseif ($type === self::$typeFile) { + $items[] = unserialize(file_get_contents($message)); + @unlink($message); + } + } + pcntl_signal_dispatch(); + // It runs the job when + // 1. Number of items reaches the batchSize. + // 2-a. Count is >0 and the current time is larger than lastInvoked + period. + // 2-b. Count is >0 and the shutdown flag is true. + if ((count($items) >= $this->batchSize) + || (count($items) > 0 + && (microtime(true) > $lastInvoked + $this->callPeriod + || $this->shutdown))) { + printf( + 'Running the job with %d items' . PHP_EOL, + count($items) + ); + $this->flush($items); + $items = []; + $lastInvoked = microtime(true); + } + gc_collect_cycles(); + if ($this->shutdown) { + return; + } + } } /** - * @return string + * Finish any pending activity for this job. + * + * @param array $items + * @return bool */ - public function getIdentifier() + public function flush(array $items = []) { - return $this->identifier; + if (!call_user_func_array($this->func, [$items])) { + $this->handleFailure($this->id, $items); + return false; + } + return true; } /** + * Returns the period in seconds from the last execution to force + * executing the job. + * * @return float */ public function getCallPeriod() @@ -147,26 +181,12 @@ public function getCallPeriod() } /** + * Returns the batch size. + * * @return int */ public function getBatchSize() { return $this->batchSize; } - - /** - * @return int - */ - public function getWorkerNum() - { - return $this->workerNum; - } - - /** - * @return string - */ - public function getBootstrapFile() - { - return $this->bootstrapFile; - } } diff --git a/src/Core/Batch/BatchRunner.php b/src/Core/Batch/BatchRunner.php index 4a2c60dd48be..ed38d8e8f330 100644 --- a/src/Core/Batch/BatchRunner.php +++ b/src/Core/Batch/BatchRunner.php @@ -33,7 +33,7 @@ class BatchRunner use SysvTrait; /** - * @var BatchConfig + * @var JobConfig */ private $config; @@ -86,7 +86,7 @@ public function __construct( * @type int $batchSize The size of the batch. * @type float $callPeriod The period in seconds from the last execution * to force executing the job. - * @type int $workerNum The number of child processes. It only takes + * @type int $numWorkers The number of child processes. It only takes * effect with the {@see \Google\Cloud\Core\Batch\BatchDaemon}. * @type string $bootstrapFile A file to load before executing the * job. It's needed for registering global functions. @@ -105,7 +105,12 @@ public function registerJob($identifier, $func, array $options = []) return false; } $this->config = $this->configStorage->load(); - $this->config->registerJob($identifier, $func, $options); + $this->config->registerJob( + $identifier, + function ($id) use ($identifier, $func, $options) { + return new BatchJob($identifier, $func, $id, $options); + } + ); try { $result = $this->configStorage->save($this->config); @@ -132,7 +137,7 @@ public function submitItem($identifier, $item) "The identifier does not exist: $identifier" ); } - $idNum = $job->getIdnum(); + $idNum = $job->id(); return $this->processor->submit($item, $idNum); } diff --git a/src/Core/Batch/BatchTrait.php b/src/Core/Batch/BatchTrait.php index 80adf17ba8bb..6e7a788d19fd 100644 --- a/src/Core/Batch/BatchTrait.php +++ b/src/Core/Batch/BatchTrait.php @@ -29,15 +29,12 @@ */ trait BatchTrait { - /** - * @var array - */ - private $batchOptions; + use SerializableClientTrait; /** * @var array */ - private $clientConfig; + private $batchOptions; /** * @var BatchRunner @@ -64,11 +61,6 @@ trait BatchTrait */ private $debugOutputResource; - /** - * @var ClosureSerializerInterface|null - */ - private $closureSerializer; - /** * Flushes items in the batch queue that have yet to be delivered. Please * note this will have no effect when using the batch daemon. @@ -79,7 +71,7 @@ public function flush() { $id = $this->batchRunner ->getJobFromId($this->identifier) - ->getIdNum(); + ->id(); return $this->batchRunner ->getProcessor() @@ -153,13 +145,14 @@ protected abstract function getCallback(); * more details. * **Defaults to** ['batchSize' => 1000, * 'callPeriod' => 2.0, - * 'workerNum' => 2]. + * 'numWorkers' => 2]. * @type array $clientConfig A config used to construct the client upon * which requests will be made. * @type BatchRunner $batchRunner A BatchRunner object. Mainly used for * the tests to inject a mock. **Defaults to** a newly created * BatchRunner. - * @type string $identifier An identifier for the batch job. + * @type string $identifier An identifier for the batch job. This + * value must be unique across all job configs. * @type string $batchMethod The name of the batch method used to * deliver items. * @type ClosureSerializerInterface $closureSerializer An implementation @@ -171,7 +164,7 @@ protected abstract function getCallback(); * } * @throws \InvalidArgumentException */ - private function setCommonBatchProperties(array $options) + private function setCommonBatchProperties(array $options = []) { if (!isset($options['identifier'])) { throw new \InvalidArgumentException( @@ -185,9 +178,7 @@ private function setCommonBatchProperties(array $options) ); } - $this->closureSerializer = isset($options['closureSerializer']) - ? $options['closureSerializer'] - : $this->getDefaultClosureSerializer(); + $this->setSerializableClientOptions($options); $this->batchMethod = $options['batchMethod']; $this->identifier = $options['identifier']; $this->debugOutputResource = isset($options['debugOutputResource']) @@ -196,14 +187,13 @@ private function setCommonBatchProperties(array $options) $this->debugOutput = isset($options['debugOutput']) ? $options['debugOutput'] : false; - $this->clientConfig = $this->getWrappedClientConfig($options); $batchOptions = isset($options['batchOptions']) ? $options['batchOptions'] : []; $this->batchOptions = $batchOptions + [ 'batchSize' => 1000, 'callPeriod' => 2.0, - 'workerNum' => 2 + 'numWorkers' => 2 ]; $this->batchRunner = isset($options['batchRunner']) ? $options['batchRunner'] @@ -214,43 +204,4 @@ private function setCommonBatchProperties(array $options) $this->batchOptions ); } - - /** - * @param array $options - * @return array - */ - private function getWrappedClientConfig(array $options) - { - $config = isset($options['clientConfig']) - ? $options['clientConfig'] - : []; - - if ($config && $this->closureSerializer) { - $this->closureSerializer->wrapClosures($config); - } - - return $config; - } - - /** - * @return array - */ - private function getUnwrappedClientConfig() - { - if ($this->clientConfig && $this->closureSerializer) { - $this->closureSerializer->unwrapClosures($this->clientConfig); - } - - return $this->clientConfig; - } - - /** - * @return ClosureSerializerInterface|null - */ - private function getDefaultClosureSerializer() - { - if (class_exists(SerializableClosure::class)) { - return new OpisClosureSerializer(); - } - } } diff --git a/src/Core/Batch/ConfigStorageInterface.php b/src/Core/Batch/ConfigStorageInterface.php index a0824d63de84..c774e92bb914 100644 --- a/src/Core/Batch/ConfigStorageInterface.php +++ b/src/Core/Batch/ConfigStorageInterface.php @@ -42,22 +42,22 @@ public function lock(); public function unlock(); /** - * saves the BatchConfig to the storage - * @param BatchConfig $config A BatchConfig to save. + * saves the JobConfig to the storage + * @param JobConfig $config A JobConfig to save. * @return bool true on success, false on failure */ - public function save(BatchConfig $config); + public function save(JobConfig $config); /** - * loads the BatchConfig from the storage + * loads the JobConfig from the storage * - * @return BatchConfig - * @throws \RuntimeException when failed to load the BatchConfig. + * @return JobConfig + * @throws \RuntimeException when failed to load the JobConfig. */ public function load(); /** - * Clear the BatchConfig from storage. + * Clear the JobConfig from storage. */ public function clear(); } diff --git a/src/Core/Batch/InMemoryConfigStorage.php b/src/Core/Batch/InMemoryConfigStorage.php index 36f75b7b840c..b69acb7846a7 100644 --- a/src/Core/Batch/InMemoryConfigStorage.php +++ b/src/Core/Batch/InMemoryConfigStorage.php @@ -31,7 +31,7 @@ final class InMemoryConfigStorage implements { use HandleFailureTrait; - /* @var BatchConfig */ + /* @var JobConfig */ private $config; /* @var array */ @@ -87,7 +87,7 @@ private function __wakeup() */ private function __construct() { - $this->config = new BatchConfig(); + $this->config = new JobConfig(); $this->created = microtime(true); $this->initFailureFile(); $this->hasShutdownHookRegistered = false; @@ -114,22 +114,22 @@ public function unlock() } /** - * Save the given BatchConfig. + * Save the given JobConfig. * - * @param BatchConfig $config A BatchConfig to save. + * @param JobConfig $config A JobConfig to save. * @return bool */ - public function save(BatchConfig $config) + public function save(JobConfig $config) { $this->config = $config; return true; } /** - * Load a BatchConfig from the storage. + * Load a JobConfig from the storage. * - * @return BatchConfig - * @throws \RuntimeException when failed to load the BatchConfig. + * @return JobConfig + * @throws \RuntimeException when failed to load the JobConfig. */ public function load() { @@ -137,11 +137,11 @@ public function load() } /** - * Clear the BatchConfig from storage. + * Clear the JobConfig from storage. */ public function clear() { - $this->config = new BatchConfig(); + $this->config = new JobConfig(); } /** @@ -191,7 +191,7 @@ public function flush($idNum) if (isset($this->items[$idNum])) { $job = $this->config->getJobFromIdNum($idNum); - if (!$job->run($this->items[$idNum])) { + if (!$job->flush($this->items[$idNum])) { $this->handleFailure($idNum, $this->items[$idNum]); } diff --git a/src/Core/Batch/InterruptTrait.php b/src/Core/Batch/InterruptTrait.php new file mode 100644 index 000000000000..70a7f0dd88ff --- /dev/null +++ b/src/Core/Batch/InterruptTrait.php @@ -0,0 +1,60 @@ +shutdown = true; + break; + } + } +} diff --git a/src/Core/Batch/BatchConfig.php b/src/Core/Batch/JobConfig.php similarity index 53% rename from src/Core/Batch/BatchConfig.php rename to src/Core/Batch/JobConfig.php index 0160ecc4d1e5..b385fcd2f8fb 100644 --- a/src/Core/Batch/BatchConfig.php +++ b/src/Core/Batch/JobConfig.php @@ -25,33 +25,34 @@ * incompatible ways. Please use with caution, and test thoroughly when * upgrading. */ -class BatchConfig +class JobConfig { /** - * @var BatchJob[] + * @var array Associative array of JobInterface instances keyed by + * identifier. */ private $jobs = []; /** - * @var array + * @var array[string]int Associative array of job identifier to job id. */ - private $idmap = []; + private $identifierToId = []; /** - * @var array + * @var array[int]string Associative array of job id to job identifier. */ - private $idmap_reverse = []; + private $idToIdentifier = []; /** * Get the job with the given identifier. * * @param string $identifier Unique identifier of the job. * - * @return BatchJob|null + * @return JobInterface|null */ public function getJobFromId($identifier) { - return array_key_exists($identifier, $this->idmap) + return array_key_exists($identifier, $this->identifierToId) ? $this->jobs[$identifier] : null; } @@ -61,12 +62,12 @@ public function getJobFromId($identifier) * * @param int $idNum A numeric id of the job. * - * @return BatchJob|null + * @return JobInterface|null */ public function getJobFromIdNum($idNum) { - return array_key_exists($idNum, $this->idmap_reverse) - ? $this->jobs[$this->idmap_reverse[$idNum]] + return array_key_exists($idNum, $this->idToIdentifier) + ? $this->jobs[$this->idToIdentifier[$idNum]] : null; } @@ -74,42 +75,29 @@ public function getJobFromIdNum($idNum) * Register a job for executing in batch. * * @param string $identifier Unique identifier of the job. - * @param callable $func Any Callable except for Closure. The callable - * should accept an array of items as the first argument. - * @param array $options [optional] { - * Configuration options. - * - * @type int $batchSize The size of the batch. - * @type float $callPeriod The period in seconds from the last execution - * to force executing the job. - * @type int $workerNum The number of child processes. It only takes - * effect with the {@see \Google\Cloud\Core\Batch\BatchDaemon}. - * @type string $bootstrapFile A file to load before executing the - * job. It's needed for registering global functions. - * } + * @param callable $callback Callback that accepts the job $idNum + * and returns a JobInterface instance. * @return void */ - public function registerJob($identifier, $func, array $options = []) + public function registerJob($identifier, $callback) { - if (array_key_exists($identifier, $this->idmap)) { - $idNum = $this->idmap[$identifier]; + if (array_key_exists($identifier, $this->identifierToId)) { + $idNum = $this->identifierToId[$identifier]; } else { - $idNum = count($this->idmap) + 1; - $this->idmap_reverse[$idNum] = $identifier; + $idNum = count($this->identifierToId) + 1; + $this->idToIdentifier[$idNum] = $identifier; } - $this->jobs[$identifier] = new BatchJob( - $identifier, - $func, - $idNum, - $options + $this->jobs[$identifier] = call_user_func( + $callback, + $idNum ); - $this->idmap[$identifier] = $idNum; + $this->identifierToId[$identifier] = $idNum; } /** - * Get all the jobs. + * Get all the jobs indexed by the job's identifier. * - * @return BatchJob[] + * @return array[string]JobInterface */ public function getJobs() { diff --git a/src/Core/Batch/JobInterface.php b/src/Core/Batch/JobInterface.php new file mode 100644 index 000000000000..4475aee8b7d5 --- /dev/null +++ b/src/Core/Batch/JobInterface.php @@ -0,0 +1,57 @@ +identifier; + } + + /** + * Return the job id + * + * @return int + */ + public function id() + { + return $this->id; + } + + /** + * Returns the number of workers for this job. **Defaults to* 1. + * + * @return int + */ + public function numWorkers() + { + return $this->numWorkers; + } + + /** + * Returns the optional file required to run this job. + * + * @return string|null + */ + public function bootstrapFile() + { + return $this->bootstrapFile; + } + + /** + * Runs the job loop. This is expected to be a blocking call. + */ + abstract public function run(); + + /** + * Finish any pending activity for this job. + * + * @param array $items + * @return bool + */ + public function flush(array $items = []) + { + return false; + } +} diff --git a/src/Core/Batch/SerializableClientTrait.php b/src/Core/Batch/SerializableClientTrait.php new file mode 100644 index 000000000000..54bedf0ffd4d --- /dev/null +++ b/src/Core/Batch/SerializableClientTrait.php @@ -0,0 +1,106 @@ + null, + 'clientConfig' => [] + ]; + $this->closureSerializer = isset($options['closureSerializer']) + ? $options['closureSerializer'] + : $this->getDefaultClosureSerializer(); + $this->setWrappedClientConfig($options); + } + + /** + * @param array $options + */ + private function setWrappedClientConfig(array $options) + { + $config = isset($options['clientConfig']) + ? $options['clientConfig'] + : []; + + if ($config && $this->closureSerializer) { + $this->closureSerializer->wrapClosures($config); + } + + $this->clientConfig = $config; + } + + /** + * @return array + */ + private function getUnwrappedClientConfig() + { + if ($this->clientConfig && $this->closureSerializer) { + $this->closureSerializer->unwrapClosures($this->clientConfig); + } + + return $this->clientConfig; + } + + /** + * @return ClosureSerializerInterface|null + */ + private function getDefaultClosureSerializer() + { + if (class_exists(SerializableClosure::class)) { + return new OpisClosureSerializer(); + } + } +} diff --git a/src/Core/Batch/SimpleJob.php b/src/Core/Batch/SimpleJob.php new file mode 100644 index 000000000000..96cb75af0e6a --- /dev/null +++ b/src/Core/Batch/SimpleJob.php @@ -0,0 +1,78 @@ +identifier = $identifier; + $this->func = $func; + $this->id = $id; + + $options += [ + 'bootstrapFile' => null, + 'numWorkers' => 1 + ]; + $this->numWorkers = $options['numWorkers']; + $this->bootstrapFile = $options['bootstrapFile']; + } + + /** + * Runs the job loop. This is expected to be a blocking call. + */ + public function run() + { + if ($this->bootstrapFile) { + require_once $this->bootstrapFile; + } + call_user_func($this->func); + } +} diff --git a/src/Core/Batch/SimpleJobTrait.php b/src/Core/Batch/SimpleJobTrait.php new file mode 100644 index 000000000000..196e38fa239c --- /dev/null +++ b/src/Core/Batch/SimpleJobTrait.php @@ -0,0 +1,105 @@ + null, + ]; + + $this->setSerializableClientOptions($options); + $identifier = $options['identifier']; + $configStorage = $options['configStorage'] ?: $this->defaultConfigStorage(); + + $result = $configStorage->lock(); + if ($result === false) { + return false; + } + $config = $configStorage->load(); + $config->registerJob( + $identifier, + function ($id) use ($identifier, $options) { + return new SimpleJob($identifier, [$this, 'run'], $id, $options); + } + ); + try { + $result = $configStorage->save($config); + } finally { + $configStorage->unlock(); + } + return $result; + } + + private function defaultConfigStorage() + { + if ($this->isSysvIPCLoaded() && $this->isDaemonRunning()) { + return new SysvConfigStorage(); + } else { + return InMemoryConfigStorage::getInstance(); + } + } +} diff --git a/src/Core/Batch/SysvConfigStorage.php b/src/Core/Batch/SysvConfigStorage.php index 778272c79591..ecc5e5e2f0d1 100644 --- a/src/Core/Batch/SysvConfigStorage.php +++ b/src/Core/Batch/SysvConfigStorage.php @@ -65,13 +65,13 @@ public function unlock() } /** - * Save the given BatchConfig. + * Save the given JobConfig. * - * @param BatchConfig $config A BatchConfig to save. + * @param JobConfig $config A JobConfig to save. * @return bool * @throws \RuntimeException when failed to attach to the shared memory or serialization fails */ - public function save(BatchConfig $config) + public function save(JobConfig $config) { $shmid = shm_attach($this->sysvKey); if ($shmid === false) { @@ -93,9 +93,9 @@ public function save(BatchConfig $config) } /** - * Load a BatchConfig from the storage. + * Load a JobConfig from the storage. * - * @return BatchConfig + * @return JobConfig * @throws \RuntimeException when failed to attach to the shared memory or deserialization fails */ public function load() @@ -107,7 +107,7 @@ public function load() ); } if (! shm_has_var($shmid, self::VAR_KEY)) { - $result = new BatchConfig(); + $result = new JobConfig(); } else { $result = shm_get_var($shmid, self::VAR_KEY); } @@ -122,7 +122,7 @@ public function load() } /** - * Clear the BatchConfig from storage. + * Clear the JobConfig from storage. */ public function clear() { diff --git a/src/Core/SysvTrait.php b/src/Core/SysvTrait.php index 10a67e13c5c1..decb40261f48 100644 --- a/src/Core/SysvTrait.php +++ b/src/Core/SysvTrait.php @@ -28,6 +28,8 @@ trait SysvTrait { private static $productionKey = 'P'; + private static $typeDirect = 1; + private static $typeFile = 2; /** * Create a SystemV IPC key for the given id number. diff --git a/src/Core/bin/google-cloud-batch b/src/Core/bin/google-cloud-batch index 7e0a161f4937..a2cbad030388 100755 --- a/src/Core/bin/google-cloud-batch +++ b/src/Core/bin/google-cloud-batch @@ -44,10 +44,11 @@ if (count($argv) < 2) { if ($argv[1] === 'daemon') { $daemon = new BatchDaemon(__FILE__); if (count($argv) == 2) { - $daemon->runParent(); + $daemon->run(); } else { $idNum = (int) $argv[2]; - $daemon->runChild($idNum); + $job = $daemon->job($idNum); + $job->run(); } } elseif ($argv[1] === 'retry') { $retry = new Retry(); diff --git a/src/Debugger/Agent.php b/src/Debugger/Agent.php index d277e4ff4adf..1536d296869d 100644 --- a/src/Debugger/Agent.php +++ b/src/Debugger/Agent.php @@ -19,6 +19,8 @@ use Google\Cloud\Core\Batch\BatchRunner; use Google\Cloud\Core\Batch\BatchTrait; +use Google\Cloud\Core\ExponentialBackoff; +use Google\Cloud\Core\Exceptions\ServiceException; use Google\Cloud\Core\SysvTrait; use Google\Cloud\Debugger\BreakpointStorage\BreakpointStorageInterface; use Google\Cloud\Debugger\BreakpointStorage\FileBreakpointStorage; @@ -92,8 +94,24 @@ public function __construct(array $options = []) $storage = isset($options['storage']) ? $options['storage'] : $this->defaultStorage(); + + $this->sourceRoot = isset($options['sourceRoot']) + ? $options['sourceRoot'] + : dirname(debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 1)[0]['file']); + + $daemon = new Daemon([ + 'sourceRoot' => $this->sourceRoot, + 'storage' => $storage + ]); + list($this->debuggeeId, $breakpoints) = $storage->load(); + // skip starting the Agent unless the Daemon has already started and + // registered the debuggee. + if (empty($this->debuggeeId)) { + return; + } + $this->setCommonBatchProperties($options + [ 'identifier' => 'stackdriver-debugger', 'batchMethod' => 'insertBatch' @@ -105,10 +123,6 @@ public function __construct(array $options = []) ? $options['logger'] : $this->defaultLogger(); - $this->sourceRoot = isset($options['sourceRoot']) - ? $options['sourceRoot'] - : dirname(debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 1)[0]['file']); - if (empty($breakpoints)) { return; } @@ -183,12 +197,32 @@ public function handleSnapshot(array $snapshot) } } - protected function getCallback() + /** + * Callback for batch runner to report a breakpoint. + * + * @access private + * @param Breakpoint[] $breakpoints + */ + public function reportBreakpoints(array $breakpoints) { if (!isset(self::$debuggee)) { self::$debuggee = $this->defaultDebuggee(); } - return [self::$debuggee, 'updateBreakpointBatch']; + foreach ($breakpoints as $breakpoint) { + $backoff = new ExponentialBackoff(); + try { + $backoff->execute(function () use ($breakpoint) { + self::$debuggee->updateBreakpoint($breakpoint); + }); + } catch (ServiceException $e) { + // Ignore this error for now + } + } + } + + protected function getCallback() + { + return [$this, 'reportBreakpoints']; } private function defaultStorage() diff --git a/src/Debugger/Daemon.php b/src/Debugger/Daemon.php index 87f92775aad6..2cf4867c087e 100644 --- a/src/Debugger/Daemon.php +++ b/src/Debugger/Daemon.php @@ -17,10 +17,13 @@ namespace Google\Cloud\Debugger; +use Google\Cloud\Core\Batch\SimpleJobTrait; +use Google\Cloud\Core\Batch\SerializableClientTrait; use Google\Cloud\Core\SysvTrait; use Google\Cloud\Core\Report\MetadataProviderInterface; use Google\Cloud\Core\Report\MetadataProviderUtils; use Google\Cloud\Core\Exception\ConflictException; +use Google\Cloud\Core\ExponentialBackoff; use Google\Cloud\Debugger\BreakpointStorage\BreakpointStorageInterface; use Google\Cloud\Debugger\BreakpointStorage\FileBreakpointStorage; use Google\Cloud\Debugger\BreakpointStorage\SysvBreakpointStorage; @@ -35,13 +38,13 @@ * ``` * use Google\Cloud\Debugger\Daemon; * - * $daemon = new Daemon('/path/to/source/root'); + * $daemon = new Daemon(); * $daemon->run(); * ``` */ class Daemon { - use SysvTrait; + use SimpleJobTrait; /** * @var Debuggee @@ -58,23 +61,48 @@ class Daemon */ private $storage; + /** + * @var array Source context configuration. + */ + private $extSourceContext; + + /** + * @var string The uniquifier for this daemon's debuggee. + */ + private $uniquifier; + + /** + * @var string The description for this daemon's debuggee. + */ + private $description; + + /** + * @var array A set of custom debuggee properties, populated by the agent, + * to be displayed to the user. + */ + private $labels; + /** * Creates a new Daemon instance. * - * @param string $sourceRoot The full path to the source root * @param array $options [optional] { * Configuration options. * - * @type DebuggerClient $client A DebuggerClient to use. **Defaults - * to** a new DebuggerClient. - * @type array $extSourceContext The source code identifier. **Defaults + * @type string $sourceRoot The full path to the source root + * @type array $clientConfig The options to instantiate the default + * DebuggerClient. + * {@see Google\Cloud\Debugger\DebuggerClient::__construct()} + * for the available options. + * @type array $sourceContext The source code identifier. **Defaults * to** values autodetected from the environment. + * @type array $extSourceContext The source code identifier. **Defaults + * to** the $sourceContext option. * @type string $uniquifier A string when uniquely identifies this * debuggee. **Defaults to** a value autodetected from the * environment. * @type string $description A display name for the debuggee in the - * Stackdriver Debugger UI. **Defaults to** a value detected - * from the environment. + * Stackdriver Debugger UI. **Defaults to** a value + * autodetected from the environment. * @type BreakpointStorageInterface $storage The breakpoint storage * mechanism to use. **Defaults to** a new SysvBreakpointStorage * instance. @@ -84,49 +112,47 @@ class Daemon * @type MetadataProviderInterface $metadataProvider **Defaults to** An * automatically chosen provider, based on detected environment * settings. + * @type ClosureSerializerInterface $closureSerializer An implementation + * responsible for serializing closures used in the + * `$clientConfig`. This is especially important when using the + * batch daemon. **Defaults to** + * {@see Google\Cloud\Core\Batch\OpisClosureSerializer} if the + * `opis/closure` library is installed. * } */ - public function __construct($sourceRoot, array $options = []) + public function __construct(array $options = []) { - $client = array_key_exists('client', $options) - ? $options['client'] - : new DebuggerClient(); - $options += [ + 'sourceRoot' => '.', 'sourceContext' => [], 'extSourceContext' => [], 'uniquifier' => null, 'description' => null, + 'debuggee' => null, 'labels' => null, 'metadataProvider' => null ]; - $this->sourceRoot = realpath($sourceRoot); - + $this->setSerializableClientOptions($options); + $this->sourceRoot = realpath($options['sourceRoot']); $sourceContext = $options['sourceContext'] ?: $this->defaultSourceContext(); - $extSourceContext = $options['extSourceContext']; - if (!$extSourceContext && $sourceContext) { - $extSourceContext = [ + $this->extSourceContext = $options['extSourceContext']; + if (!$this->extSourceContext && $sourceContext) { + $this->extSourceContext = [ 'context' => $sourceContext ]; } - $uniquifier = $options['uniquifier'] ?: $this->defaultUniquifier(); - $description = $options['description'] ?: $this->defaultDescription(); - $labels = $options['labels'] ?: $this->defaultLabels($options['metadataProvider']); - - $this->debuggee = $client->debuggee(null, [ - 'uniquifier' => $uniquifier, - 'description' => $description, - 'extSourceContexts' => $extSourceContext ? [$extSourceContext] : [], - 'labels' => $labels - ]); - - $this->debuggee->register(); - + $this->uniquifier = $options['uniquifier']; + $this->description = $options['description'] ?: $this->defaultDescription(); + $this->labels = $options['labels'] ?: $this->defaultLabels($options['metadataProvider']); $this->storage = array_key_exists('storage', $options) ? $options['storage'] : $this->defaultStorage(); + + $this->setSimpleJobProperties($options + [ + 'identifier' => 'debugger-daemon' + ]); } /** @@ -139,24 +165,43 @@ public function __construct($sourceRoot, array $options = []) * $daemon->run(); * ``` */ - public function run() + public function run(DebuggerClient $client = null) { - $resp = $this->debuggee->breakpointsWithWaitToken(); - $this->setBreakpoints($resp['breakpoints']); + $client = $client ?: $this->defaultClient(); + $extSourceContexts = $this->extSourceContext ? [$this->extSourceContext] : []; + $debuggee = $client->debuggee(null, [ + 'uniquifier' => $this->uniquifier ?: $this->defaultUniquifier(), + 'description' => $this->description, + 'extSourceContexts' => $extSourceContexts, + 'labels' => $this->labels + ]); + $debuggee->register(); + + $resp = $this->fetchBreakpointsWithRetry($debuggee); + $this->setBreakpoints($debuggee, $resp['breakpoints']); while (array_key_exists('nextWaitToken', $resp)) { try { - $resp = $this->debuggee->breakpointsWithWaitToken([ + $resp = $this->fetchBreakpointsWithRetry($debuggee, [ 'waitToken' => $resp['nextWaitToken'] ]); - $this->setBreakpoints($resp['breakpoints']); + $this->setBreakpoints($debuggee, $resp['breakpoints']); } catch (ConflictException $e) { // Ignoring this exception } + gc_collect_cycles(); } } - private function setBreakpoints($breakpoints) + private function fetchBreakpointsWithRetry(Debuggee $debuggee, array $options = []) + { + $backoff = new ExponentialBackoff(); + return $backoff->execute(function () use ($debuggee, $options) { + return $debuggee->breakpointsWithWaitToken($options); + }); + } + + private function setBreakpoints(Debuggee $debuggee, $breakpoints) { $validBreakpoints = []; $invalidBreakpoints = []; @@ -170,10 +215,10 @@ private function setBreakpoints($breakpoints) } } - $this->storage->save($this->debuggee, $validBreakpoints); + $this->storage->save($debuggee, $validBreakpoints); if (!empty($invalidBreakpoints)) { - $this->debuggee->updateBreakpointBatch($invalidBreakpoints); + $debuggee->updateBreakpointBatch($invalidBreakpoints); } } @@ -211,6 +256,11 @@ private function defaultSourceContext() return []; } + private function defaultClient() + { + return new DebuggerClient($this->getUnwrappedClientConfig()); + } + private function defaultLabels(MetadataProviderInterface $metadataProvider = null) { $metadataProvider = $metadataProvider ?: MetadataProviderUtils::autoSelect($_SERVER); diff --git a/src/Debugger/Debuggee.php b/src/Debugger/Debuggee.php index a460a8680b86..358a4cec6db1 100644 --- a/src/Debugger/Debuggee.php +++ b/src/Debugger/Debuggee.php @@ -375,6 +375,9 @@ public function jsonSerialize() 'agentVersion' => $this->agentVersion, 'status' => $this->status, 'sourceContexts' => array_map(function ($esc) { + if (empty($esc)) { + return []; + } return is_array($esc) ? $esc['context'] : $esc->context(); }, $this->extSourceContexts), 'extSourceContexts' => $this->extSourceContexts diff --git a/src/Debugger/bin/google-cloud-debugger b/src/Debugger/bin/google-cloud-debugger deleted file mode 100755 index fe7db0e5b086..000000000000 --- a/src/Debugger/bin/google-cloud-debugger +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env php -run(); diff --git a/src/Debugger/composer.json b/src/Debugger/composer.json index 0518edd221f9..46dacc442cf3 100644 --- a/src/Debugger/composer.json +++ b/src/Debugger/composer.json @@ -17,9 +17,6 @@ "entry": "DebuggerClient.php" } }, - "bin": [ - "bin/google-cloud-debugger" - ], "autoload": { "psr-4": { "Google\\Cloud\\Debugger\\": "" diff --git a/src/ErrorReporting/Bootstrap.php b/src/ErrorReporting/Bootstrap.php index 862f3c790fa0..8c3f0dccd61a 100644 --- a/src/ErrorReporting/Bootstrap.php +++ b/src/ErrorReporting/Bootstrap.php @@ -29,7 +29,7 @@ public static function init(PsrLogger $psrLogger = null) 'batchEnabled' => true, 'debugOutput' => true, 'batchOptions' => [ - 'workerNum' => 2 + 'numWorkers' => 2 ] ]); register_shutdown_function([self::class, 'shutdownHandler']); diff --git a/src/Logging/LoggingClient.php b/src/Logging/LoggingClient.php index ca2e885f47ed..50e60aa00990 100644 --- a/src/Logging/LoggingClient.php +++ b/src/Logging/LoggingClient.php @@ -114,7 +114,7 @@ class LoggingClient * more details. * **Defaults to** ['batchSize' => 1000, * 'callPeriod' => 2.0, - * 'workerNum' => 2]. + * 'numWorkers' => 2]. * @type array $clientConfig Configuration options for the Logging client * used to handle processing of batch items. For valid options * please see @@ -523,7 +523,7 @@ function (array $entry) { * more details. * **Defaults to** ['batchSize' => 1000, * 'callPeriod' => 2.0, - * 'workerNum' => 2]. Applies only when + * 'numWorkers' => 2]. Applies only when * `batchEnabled` is set to `true`. Note that this option is * currently considered **experimental** and is subject to change. * @type array $clientConfig Configuration options for the Logging client diff --git a/src/Logging/PsrLogger.php b/src/Logging/PsrLogger.php index 66d1da08de7c..25d5d921df40 100644 --- a/src/Logging/PsrLogger.php +++ b/src/Logging/PsrLogger.php @@ -109,7 +109,7 @@ class PsrLogger implements LoggerInterface, \Serializable * more details. * **Defaults to** ['batchSize' => 1000, * 'callPeriod' => 2.0, - * 'workerNum' => 2]. Applies only when + * 'numWorkers' => 2]. Applies only when * `batchEnabled` is set to `true`. * @type array $clientConfig Configuration options for the Logging client * used to handle processing of batch items. For valid options diff --git a/src/PubSub/Topic.php b/src/PubSub/Topic.php index 97a34bb0ea44..047e8af1485b 100644 --- a/src/PubSub/Topic.php +++ b/src/PubSub/Topic.php @@ -370,7 +370,7 @@ public function publishBatch(array $messages, array $options = []) * more details. * **Defaults to** ['batchSize' => 1000, * 'callPeriod' => 2.0, - * 'workerNum' => 2]. + * 'numWorkers' => 2]. * @type array $clientConfig Configuration options for the PubSub client * used to handle processing of batch items. For valid options * please see diff --git a/tests/snippets/Debugger/DaemonTest.php b/tests/snippets/Debugger/DaemonTest.php index 855756fffc75..bad688c6af61 100644 --- a/tests/snippets/Debugger/DaemonTest.php +++ b/tests/snippets/Debugger/DaemonTest.php @@ -39,19 +39,20 @@ public function setUp() $this->debuggee = $this->prophesize(Debuggee::class); $this->storage = $this->prophesize(BreakpointStorageInterface::class); $this->debuggee->register()->willReturn(true); - $this->debuggee->breakpointsWithWaitToken()->willReturn(['breakpoints' => []]); + $this->debuggee->breakpointsWithWaitToken([])->willReturn(['breakpoints' => []]); $this->client->debuggee(null, Argument::any())->willReturn($this->debuggee->reveal()); } public function testClass() { $options = [ - 'client' => $this->client->reveal(), 'storage' => $this->storage->reveal() ]; $snippet = $this->snippetFromClass(Daemon::class); - $snippet->replace('new Daemon(\'/path/to/source/root\')', 'new Daemon(__DIR__, $options)'); + $snippet->replace('new Daemon()', 'new Daemon($options)'); + $snippet->replace('run()', 'run($client)'); $snippet->addLocal('options', $options); + $snippet->addLocal('client', $this->client->reveal()); $res = $snippet->invoke('daemon'); $this->assertInstanceOf(Daemon::class, $res->returnVal()); } @@ -59,12 +60,13 @@ public function testClass() public function testRun() { $options = [ - 'client' => $this->client->reveal(), 'storage' => $this->storage->reveal() ]; - $daemon = new Daemon(__DIR__, $options); + $daemon = new Daemon($options); $snippet = $this->snippetFromMethod(Daemon::class, 'run'); + $snippet->replace('run()', 'run($client)'); $snippet->addLocal('daemon', $daemon); + $snippet->addLocal('client', $this->client->reveal()); $res = $snippet->invoke('daemon'); $this->assertInstanceOf(Daemon::class, $res->returnVal()); } diff --git a/tests/system/Core/Batch/BatchRunnerTest.php b/tests/system/Core/Batch/BatchRunnerTest.php index 22c4c8d005d3..387ed8b8defc 100644 --- a/tests/system/Core/Batch/BatchRunnerTest.php +++ b/tests/system/Core/Batch/BatchRunnerTest.php @@ -110,7 +110,7 @@ public function setup() 'batch-daemon-system-test', array($myJob, 'runJob'), array( - 'workerNum' => 1, + 'numWorkers' => 1, 'batchSize' => 2, 'callPeriod' => 1, ) @@ -158,7 +158,7 @@ public function testSubmit() . '/../../../../src/Core/bin/google-cloud-batch retry'; exec($retry_command); } else { - // The in-memory implementation doesn't share the BatchConfig with + // The in-memory implementation doesn't share the JobConfig with // other processes, so we need to run retryAll in the same process. $retry = new Retry(); $retry->retryAll(); diff --git a/tests/system/Debugger/E2ETest.php b/tests/system/Debugger/E2ETest.php index 2c31b34af9a1..6f31e6010d9d 100644 --- a/tests/system/Debugger/E2ETest.php +++ b/tests/system/Debugger/E2ETest.php @@ -38,8 +38,8 @@ */ class E2ETest extends TestCase { - protected static $debuggeeId; - protected static $httpClient; + protected $debuggeeId; + protected $httpClient; use AppEngineDeploymentTrait; use EventuallyConsistentTestTrait; @@ -50,21 +50,18 @@ public static function beforeDeploy() self::$gcloudWrapper->setDir(implode(DIRECTORY_SEPARATOR, [__DIR__, 'app'])); } - public static function afterDeploy() + public function setUp() { $url = self::$gcloudWrapper->getBaseUrl(); - self::$httpClient = new Client(['base_uri' => $url]); - - $resp = self::$httpClient->get('/debuggee'); - $attempts = 0; - while ($resp->getStatusCode() != 200 && $attempts < 5) { - sleep(pow(2, $attempts)); - $resp = self::$httpClient->get('/debuggee'); - $attempts++; - } + $this->httpClient = new Client(['base_uri' => $url]); - $data = json_decode($resp->getBody()->getContents(), true); - self::$debuggeeId = $data['debuggeeId']; + $this->runEventuallyConsistentTest(function () { + $resp = $this->httpClient->get('/debuggee'); + $this->assertEquals(200, $resp->getStatusCode()); + $data = json_decode($resp->getBody()->getContents(), true); + $this->assertNotEmpty($data['debuggeeId']); + $this->debuggeeId = $data['debuggeeId']; + }, 5, true); } public static function tearDownAfterClass() @@ -118,7 +115,7 @@ public function testWithFullPath() $this->assertBreakpointCount(1); }); - $resp = self::$httpClient->get('hello/full'); + $resp = $this->httpClient->get('hello/full'); $this->assertEquals('200', $resp->getStatusCode(), 'hello/full status code'); $this->assertContains('Hello, full', $resp->getBody()->getContents()); @@ -129,7 +126,7 @@ public function testWithFullPath() private function assertBreakpointCount($count) { - $resp = self::$httpClient->get('/debuggee'); + $resp = $this->httpClient->get('/debuggee'); $data = json_decode($resp->getBody()->getContents(), true); $this->assertEquals($count, (int) $data['numBreakpoints']); } @@ -142,7 +139,7 @@ public function testWithExtraPath() $this->assertBreakpointCount(1); }); - $resp = self::$httpClient->get('hello/extra'); + $resp = $this->httpClient->get('hello/extra'); $this->assertEquals('200', $resp->getStatusCode(), 'hello/extra status code'); $this->assertContains('Hello, extra', $resp->getBody()->getContents()); @@ -159,7 +156,7 @@ public function testWithMissingPath() $this->assertBreakpointCount(1); }); - $resp = self::$httpClient->get('hello/missing'); + $resp = $this->httpClient->get('hello/missing'); $this->assertEquals('200', $resp->getStatusCode(), 'hello/missing status code'); $this->assertContains('Hello, missing', $resp->getBody()->getContents()); @@ -174,7 +171,7 @@ private function setBreakpoint($file, $line) $client = new DebuggerClient([ 'keyFilePath' => getenv('GOOGLE_CLOUD_PHP_TESTS_KEY_PATH') ]); - $debuggee = $client->debuggee(self::$debuggeeId); + $debuggee = $client->debuggee($this->debuggeeId); $breakpoint = $debuggee->setBreakpoint($file, $line); $this->assertInstanceOf(Breakpoint::class, $breakpoint); $this->assertNotNull($breakpoint->location()); diff --git a/tests/system/Debugger/app/additional-supervisord.conf b/tests/system/Debugger/app/additional-supervisord.conf deleted file mode 100644 index bd14ca20a634..000000000000 --- a/tests/system/Debugger/app/additional-supervisord.conf +++ /dev/null @@ -1,11 +0,0 @@ -[program:debugger-daemon] -command = php -d auto_prepend_file='' -d disable_functions='' /app/vendor/bin/google-cloud-debugger /app -stdout_logfile = /dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile = /dev/stderr -stderr_logfile_maxbytes=0 -user = www-data -autostart = true -autorestart = true -priority = 5 -stopwaitsecs = 20 diff --git a/tests/unit/Core/Batch/BatchJobTest.php b/tests/unit/Core/Batch/BatchJobTest.php index 3467e8c287b0..e778df3e2574 100644 --- a/tests/unit/Core/Batch/BatchJobTest.php +++ b/tests/unit/Core/Batch/BatchJobTest.php @@ -33,10 +33,10 @@ public function testDefault() $job = new BatchJob('testing', array($this, 'runJob'), 1); $this->assertEquals(100, $job->getBatchSize()); $this->assertEquals(2.0, $job->getCallPeriod()); - $this->assertEquals(1, $job->getWorkerNum()); - $this->assertNull($job->getBootstrapFile()); - $this->assertEquals(1, $job->getIdNum()); - $this->assertEquals('testing', $job->getIdentifier()); + $this->assertEquals(1, $job->numWorkers()); + $this->assertNull($job->bootstrapFile()); + $this->assertEquals(1, $job->id()); + $this->assertEquals('testing', $job->identifier()); } public function testCustom() @@ -49,24 +49,15 @@ public function testCustom() 'batchSize' => 1000, 'callPeriod' => 1.0, 'bootstrapFile' => __FILE__, - 'workerNum' => 10 + 'numWorkers' => 10 ) ); $this->assertEquals(1000, $job->getBatchSize()); $this->assertEquals(1.0, $job->getCallPeriod()); - $this->assertEquals(10, $job->getWorkerNum()); - $this->assertEquals(__FILE__, $job->getBootstrapFile()); - $this->assertEquals(1, $job->getIdNum()); - $this->assertEquals('testing', $job->getIdentifier()); - } - - public function testRun() - { - $job = new BatchJob('testing', array($this, 'runJob'), 1); - $items = array('apple', 'orange', 'banana'); - $expected = array('APPLE', 'ORANGE', 'BANANA'); - $job->run($items); - $this->assertEquals($expected, $this->items); + $this->assertEquals(10, $job->numWorkers()); + $this->assertEquals(__FILE__, $job->bootstrapFile()); + $this->assertEquals(1, $job->id()); + $this->assertEquals('testing', $job->identifier()); } /** diff --git a/tests/unit/Core/Batch/BatchRunnerTest.php b/tests/unit/Core/Batch/BatchRunnerTest.php index 86ae5d9b5748..b8655367382c 100644 --- a/tests/unit/Core/Batch/BatchRunnerTest.php +++ b/tests/unit/Core/Batch/BatchRunnerTest.php @@ -17,7 +17,7 @@ namespace Google\Cloud\Tests\Unit\Core\Batch; -use Google\Cloud\Core\Batch\BatchConfig; +use Google\Cloud\Core\Batch\JobConfig; use Google\Cloud\Core\Batch\BatchJob; use Google\Cloud\Core\Batch\BatchRunner; use Google\Cloud\Core\Batch\ConfigStorageInterface; @@ -38,7 +38,7 @@ public function setUp() { $this->configStorage = $this->prophesize(ConfigStorageInterface::class); $this->processor = $this->prophesize(ProcessItemInterface::class); - $this->batchConfig = $this->prophesize(BatchConfig::class); + $this->batchConfig = $this->prophesize(JobConfig::class); } /** @@ -90,7 +90,7 @@ public function testConstructorLoadConfig() public function testRegisterJob() { - $this->batchConfig->registerJob('test', 'myFunc', []) + $this->batchConfig->registerJob('test', Argument::type(\Closure::class)) ->shouldBeCalledTimes(1); $config = $this->batchConfig->reveal(); $this->configStorage->lock() @@ -99,7 +99,7 @@ public function testRegisterJob() $this->configStorage->load() ->willreturn($config) ->shouldBeCalledTimes(2); - $this->configStorage->save(Argument::type(BatchConfig::class)) + $this->configStorage->save(Argument::type(JobConfig::class)) ->willreturn(true) ->shouldBeCalledTimes(1); $this->configStorage->unlock() diff --git a/tests/unit/Core/Batch/BatchTraitTest.php b/tests/unit/Core/Batch/BatchTraitTest.php index b09ec1e6f96a..4f51d95a6516 100644 --- a/tests/unit/Core/Batch/BatchTraitTest.php +++ b/tests/unit/Core/Batch/BatchTraitTest.php @@ -38,7 +38,7 @@ public function testFlush() $idNum = 5; $returnVal = true; $job = $this->prophesize(BatchJob::class); - $job->getIdNum() + $job->id() ->willReturn($idNum) ->shouldBeCalledTimes(1); $processor = $this->prophesize(ProcessItemInterface::class); diff --git a/tests/unit/Core/Batch/InMemoryConfigStorageTest.php b/tests/unit/Core/Batch/InMemoryConfigStorageTest.php index 36389b2f2f2f..ec21ac028ecd 100644 --- a/tests/unit/Core/Batch/InMemoryConfigStorageTest.php +++ b/tests/unit/Core/Batch/InMemoryConfigStorageTest.php @@ -17,7 +17,8 @@ namespace Google\Cloud\Tests\Unit\Core\Batch; -use Google\Cloud\Core\Batch\BatchConfig; +use Google\Cloud\Core\Batch\BatchJob; +use Google\Cloud\Core\Batch\JobConfig; use Google\Cloud\Core\Batch\InMemoryConfigStorage; use PHPUnit\Framework\TestCase; @@ -61,7 +62,7 @@ public function testUnLock() public function testSaveAndLoad() { $configStorage = InMemoryConfigStorage::getInstance(); - $config = new BatchConfig(); + $config = new JobConfig(); $configStorage->save($config); $this->assertEquals($config, $configStorage->load()); } @@ -69,11 +70,12 @@ public function testSaveAndLoad() public function testSubmit() { $configStorage = InMemoryConfigStorage::getInstance(); - $config = new BatchConfig(); + $config = new JobConfig(); $config->registerJob( 'testSubmit', - array($this, 'runJob'), - array('batchSize' => 2) + function ($id) { + return new BatchJob('testSubmit', [$this, 'runJob'], $id, ['batchSize' => 2]); + } ); $configStorage->save($config); diff --git a/tests/unit/Core/Batch/BatchConfigTest.php b/tests/unit/Core/Batch/JobConfigTest.php similarity index 67% rename from tests/unit/Core/Batch/BatchConfigTest.php rename to tests/unit/Core/Batch/JobConfigTest.php index 14a44f19f315..7423d06620cb 100644 --- a/tests/unit/Core/Batch/BatchConfigTest.php +++ b/tests/unit/Core/Batch/JobConfigTest.php @@ -17,14 +17,15 @@ namespace Google\Cloud\Tests\Unit\Core\Batch; -use Google\Cloud\Core\Batch\BatchConfig; +use Google\Cloud\Core\Batch\JobConfig; +use Google\Cloud\Core\Batch\BatchJob; use PHPUnit\Framework\TestCase; /** * @group core * @group batch */ -class BatchConfigTest extends TestCase +class JobConfigTest extends TestCase { private $config; private $identifier; @@ -33,12 +34,14 @@ class BatchConfigTest extends TestCase public function setUp() { - $this->config = new BatchConfig(); + $this->config = new JobConfig(); $this->identifier = 'job1'; $this->func = 'myFunc'; $this->config->registerJob( $this->identifier, - $this->func, + function ($id) { + return new BatchJob($this->identifier, $this->func, $id); + }, [] ); // It must have 1 as the idNum. @@ -48,16 +51,16 @@ public function setUp() public function testGetJobFromId() { $job = $this->config->getJobFromId($this->identifier); - $this->assertEquals($this->idNum, $job->getIdNum()); - $this->assertEquals($this->identifier, $job->getIdentifier()); + $this->assertEquals($this->idNum, $job->id()); + $this->assertEquals($this->identifier, $job->identifier()); $this->assertNull($this->config->getJobFromId('bogus')); } public function testGetJobFromIdNum() { $job = $this->config->getJobFromIdNum($this->idNum); - $this->assertEquals($this->idNum, $job->getIdNum()); - $this->assertEquals($this->identifier, $job->getIdentifier()); + $this->assertEquals($this->idNum, $job->id()); + $this->assertEquals($this->identifier, $job->identifier()); $this->assertNull($this->config->getJobFromIdNum(10)); } @@ -66,13 +69,15 @@ public function testRegisterJob() $identifier = 'job2'; $this->config->registerJob( $identifier, - $this->func, + function ($id) use ($identifier) { + return new BatchJob($identifier, $this->func, $id); + }, [] ); // The idNum is 1 origin, incremented by 1 $job = $this->config->getJobFromIdNum(2); - $this->assertEquals(2, $job->getIdNum()); - $this->assertEquals($identifier, $job->getIdentifier()); + $this->assertEquals(2, $job->id()); + $this->assertEquals($identifier, $job->identifier()); } public function testGetjobs() @@ -80,12 +85,14 @@ public function testGetjobs() $identifier = 'job2'; $this->config->registerJob( $identifier, - $this->func, + function ($id) use ($identifier) { + return new BatchJob($identifier, $this->func, $id); + }, [] ); $jobs = $this->config->getJobs(); $this->assertCount(2, $jobs); - $this->assertEquals($this->idNum, $jobs[$this->identifier]->getIdNum()); - $this->assertEquals(2, $jobs[$identifier]->getIdNum()); + $this->assertEquals($this->idNum, $jobs[$this->identifier]->id()); + $this->assertEquals(2, $jobs[$identifier]->id()); } } diff --git a/tests/unit/Core/Batch/JobTraitTest.php b/tests/unit/Core/Batch/JobTraitTest.php new file mode 100644 index 000000000000..ee1e7254a260 --- /dev/null +++ b/tests/unit/Core/Batch/JobTraitTest.php @@ -0,0 +1,50 @@ +assertNull($job->identifier()); + $this->assertNull($job->id()); + $this->assertNull($job->numWorkers()); + $this->assertNull($job->bootstrapFile()); + $this->assertFalse($job->flush()); + $job->run(); + } +} + +class TestJob implements JobInterface +{ + use JobTrait; + + public function run() + { + // Do nothing + } +} diff --git a/tests/unit/Core/Batch/SimpleJobTest.php b/tests/unit/Core/Batch/SimpleJobTest.php new file mode 100644 index 000000000000..9bc30839f4d2 --- /dev/null +++ b/tests/unit/Core/Batch/SimpleJobTest.php @@ -0,0 +1,47 @@ +assertEquals('testing', $job->identifier()); + $this->assertEquals(1, $job->numWorkers()); + $this->assertFalse($job->flush()); + $this->assertNull($job->bootstrapFile()); + + $job->run(); + $this->assertTrue($this->success); + } + + public function runJob() + { + $this->success = true; + } +} diff --git a/tests/unit/Core/Batch/SimpleJobTraitTest.php b/tests/unit/Core/Batch/SimpleJobTraitTest.php new file mode 100644 index 000000000000..72b6c18b3b0a --- /dev/null +++ b/tests/unit/Core/Batch/SimpleJobTraitTest.php @@ -0,0 +1,86 @@ +setSimpleJobProperties([]); + } + + public function testRegistersConfig() + { + $storage = InMemoryConfigStorage::getInstance(); + $storage->clear(); + + $impl = new SimpleClass(); + $impl->setSimpleJobProperties([ + 'identifier' => self::ID, + 'configStorage' => $storage + ]); + + $config = $storage->load(); + $this->assertInstanceOf(JobConfig::class, $config); + $job = $config->getJobFromId(self::ID); + $this->assertInstanceOf(SimpleJob::class, $job); + + $job->run(); + $this->assertTrue($impl->hasRun()); + } +} + +class SimpleClass +{ + use SimpleJobTrait { + setSimpleJobProperties as privateSetSimpleJobProperties; + } + + private $hasRun = false; + + public function setSimpleJobProperties(array $options) + { + $this->privateSetSimpleJobProperties($options); + } + + public function run() + { + $this->hasRun = true; + } + + public function hasRun() + { + return $this->hasRun; + } +} diff --git a/tests/unit/Core/Batch/SysvConfigStorageTest.php b/tests/unit/Core/Batch/SysvConfigStorageTest.php index e82951d61cb3..f2e4e545db45 100644 --- a/tests/unit/Core/Batch/SysvConfigStorageTest.php +++ b/tests/unit/Core/Batch/SysvConfigStorageTest.php @@ -17,7 +17,8 @@ namespace Google\Cloud\Tests\Unit\Core\Batch; -use Google\Cloud\Core\Batch\BatchConfig; +use Google\Cloud\Core\Batch\BatchJob; +use Google\Cloud\Core\Batch\JobConfig; use Google\Cloud\Core\Batch\SysvConfigStorage; use Google\Cloud\Core\SysvTrait; use PHPUnit\Framework\TestCase; @@ -49,7 +50,7 @@ public function testLockAndUnlock() public function testSaveAndLoad() { - $config = new BatchConfig(); + $config = new JobConfig(); $this->storage->save($config); $this->assertEquals($config, $this->storage->load()); } @@ -57,8 +58,10 @@ public function testSaveAndLoad() public function testSaveBadConfig() { $object = new TestSerializableObjectWithClosure(); - $config = new BatchConfig(); - $config->registerJob('badConfig', [$object, 'callback']); + $config = new JobConfig(); + $config->registerJob('badConfig', function ($id) use ($object) { + return new BatchJob('badConfig', $id, [$object, 'callback']); + }); try { $this->storage->save($config); diff --git a/tests/unit/Debugger/DaemonTest.php b/tests/unit/Debugger/DaemonTest.php index af82fa07ddf8..a0fc119a3a3a 100644 --- a/tests/unit/Debugger/DaemonTest.php +++ b/tests/unit/Debugger/DaemonTest.php @@ -44,42 +44,57 @@ public function setUp() public function testSpecifyUniquifier() { - $this->debuggee->register(Argument::any())->shouldBeCalled(); + $resp = [ + 'breakpoints' => [] + ]; + $this->debuggee->register(Argument::any()) + ->shouldBeCalled(); + $this->debuggee->breakpointsWithWaitToken([]) + ->willReturn($resp); $this->client->debuggee(null, Argument::withEntry('uniquifier', 'some uniquifier')) ->willReturn($this->debuggee->reveal())->shouldBeCalled(); - $daemon = new Daemon('.', [ - 'client' => $this->client->reveal(), + $daemon = new Daemon([ 'storage' => $this->storage->reveal(), 'uniquifier' => 'some uniquifier' ]); + $daemon->run($this->client->reveal()); } public function testGeneratesDefaultUniquifier() { - $this->debuggee->register(Argument::any())->shouldBeCalled(); + $resp = [ + 'breakpoints' => [] + ]; + $this->debuggee->register(Argument::any()) + ->shouldBeCalled(); + $this->debuggee->breakpointsWithWaitToken([]) + ->willReturn($resp); $this->client->debuggee(null, Argument::that(function ($options) { return preg_match('/[a-z0-9]{32}/', $options['uniquifier']); }))->willReturn($this->debuggee->reveal())->shouldBeCalled(); - $root = implode(DIRECTORY_SEPARATOR, [__DIR__, 'data']); - $daemon = new Daemon($root, [ - 'client' => $this->client->reveal(), + $daemon = new Daemon([ + 'sourceRoot' => implode(DIRECTORY_SEPARATOR, [__DIR__, 'data']), 'storage' => $this->storage->reveal() ]); + $daemon->run($this->client->reveal()); } public function testSpecifyDescription() { $this->debuggee->register(Argument::any())->shouldBeCalled(); + $this->debuggee->breakpointsWithWaitToken([]) + ->willReturn(['breakpoints' => []]); $this->client->debuggee(null, Argument::withEntry('description', 'some description')) ->willReturn($this->debuggee->reveal())->shouldBeCalled(); - $daemon = new Daemon('.', [ + $daemon = new Daemon([ 'client' => $this->client->reveal(), 'storage' => $this->storage->reveal(), 'description' => 'some description' ]); + $daemon->run($this->client->reveal()); } public function testSpecifyExtSourceContext() @@ -93,27 +108,35 @@ public function testSpecifyExtSourceContext() ], 'labels' => [] ]; - $this->debuggee->register(Argument::any())->shouldBeCalled(); + $resp = [ + 'breakpoints' => [] + ]; + $this->debuggee->register(Argument::any()) + ->shouldBeCalled(); + $this->debuggee->breakpointsWithWaitToken([]) + ->willReturn($resp); $this->client->debuggee(null, Argument::withEntry('extSourceContexts', [$context])) ->willReturn($this->debuggee->reveal())->shouldBeCalled(); - $daemon = new Daemon('.', [ - 'client' => $this->client->reveal(), + $daemon = new Daemon([ 'storage' => $this->storage->reveal(), 'extSourceContext' => $context ]); + $daemon->run($this->client->reveal()); } public function testEmptyDefaultSourceContext() { $this->debuggee->register(Argument::any())->shouldBeCalled(); + $this->debuggee->breakpointsWithWaitToken([]) + ->willReturn(['breakpoints' => []]); $this->client->debuggee(null, Argument::withEntry('extSourceContexts', [])) ->willReturn($this->debuggee->reveal())->shouldBeCalled(); - $daemon = new Daemon('.', [ - 'client' => $this->client->reveal(), + $daemon = new Daemon([ 'storage' => $this->storage->reveal() ]); + $daemon->run($this->client->reveal()); } public function testDefaultSourceContext() @@ -127,14 +150,17 @@ public function testDefaultSourceContext() ] ]; $this->debuggee->register(Argument::any())->shouldBeCalled(); + $this->debuggee->breakpointsWithWaitToken([]) + ->willReturn(['breakpoints' => []]); $this->client->debuggee(null, Argument::withEntry('extSourceContexts', [$expectedSourceContext])) ->willReturn($this->debuggee->reveal())->shouldBeCalled(); $root = implode(DIRECTORY_SEPARATOR, [__DIR__, 'data']); - $daemon = new Daemon($root, [ - 'client' => $this->client->reveal(), + $daemon = new Daemon([ + 'sourceRoot' => $root, 'storage' => $this->storage->reveal() ]); + $daemon->run($this->client->reveal()); } public function testFetchesBreakpoints() @@ -144,19 +170,17 @@ public function testFetchesBreakpoints() ]; $this->debuggee->register(Argument::any()) ->shouldBeCalled(); - $this->debuggee->breakpointsWithWaitToken() + $this->debuggee->breakpointsWithWaitToken([]) ->willReturn($resp); $this->debuggee->updateBreakpointBatch(Argument::any()) ->willReturn(true); $this->client->debuggee(null, Argument::any()) - ->willReturn($this->debuggee->reveal()) - ->shouldBeCalled(); + ->willReturn($this->debuggee->reveal())->shouldBeCalled(); - $daemon = new Daemon('.', [ - 'client' => $this->client->reveal(), + $daemon = new Daemon([ 'storage' => $this->storage->reveal() ]); - $daemon->run(); + $daemon->run($this->client->reveal()); } public function testDetectsLabelsFromEnvironment() @@ -169,14 +193,18 @@ public function testDetectsLabelsFromEnvironment() ]; $this->debuggee->register(Argument::any()) ->shouldBeCalled(); + $this->debuggee->breakpointsWithWaitToken([]) + ->willReturn(['breakpoints' => []]); + $this->debuggee->updateBreakpointBatch(Argument::any()) + ->willReturn(true); $this->client->debuggee(null, Argument::withEntry('labels', $expectedLabels)) ->willReturn($this->debuggee->reveal()) ->shouldBeCalled(); - $daemon = new Daemon('.', [ + $daemon = new Daemon([ 'metadataProvider' => $provider, - 'client' => $this->client->reveal(), 'storage' => $this->storage->reveal() ]); + $daemon->run($this->client->reveal()); } }