- Remove Sqlite symbol table. It's slow and inferior to JsonSymbolTable.

- Update json_encode to substitute utf-8 issues in the JSON and to throw exceptions on error. - Use a single generator function for indexing multiple paths. - Clean up some variable names to make code more readable.
BambooHR · Dec 20, 2023 · 114eca8 · 114eca8
1 parent 6611d78
commit 114eca8
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 594 deletions.
diff --git a/composer.json b/composer.json
@@ -19,7 +19,8 @@
     "webmozart/glob":"^4.1.0",
     "n98/junit-xml":"^1.0.0",
     "seld/jsonlint": "^1.7",
-    "ext-pcntl": "*"
+    "ext-pcntl": "*",
+    "ext-sockets": "*"
   },
   "require-dev": {
     "phpunit/phpunit": "^9.0"

diff --git a/src/CommandLineRunner.php b/src/CommandLineRunner.php
@@ -36,11 +36,9 @@ public function usage() {
        -a                     = run the \"analyze\" operation
 
        -i                     = run the \"index\" operation.
-                                Defaults to yes if using in memory index.
-
-       -s                     = prefer sqlite index
+                                Defaults to yes if using in memory index.       
         
-       -j                     = prefer json index (new)
+       -j                     = prefer json index
 
        -m                     = prefer in memory index (only available when -n=1 and -p=1/1)
 

diff --git a/src/Config.php b/src/Config.php
@@ -153,22 +153,18 @@ public function __construct($argv) {
 		}
 
 		if ($this->processes > 1 && $this->preferredTable == self::MEMORY_SYMBOL_TABLE) {
-			$this->preferredTable = self::SQLITE_SYMBOL_TABLE;
+			$this->preferredTable = self::JSON_SYMBOL_TABLE;
 		}
 
-		if ($this->preferredTable == self::SQLITE_SYMBOL_TABLE || $this->preferredTable == self::JSON_SYMBOL_TABLE) {
+		if ($this->preferredTable == self::JSON_SYMBOL_TABLE) {
 			if (!file_exists($this->getSymbolTableFile())) {
 				$this->forceIndex = true;
 			}
 			if ($this->forceIndex && file_exists($this->getSymbolTableFile())) {
 				unlink($this->getSymbolTableFile());
 			}
 
-			if ($this->preferredTable == self::JSON_SYMBOL_TABLE) {
-				$this->symbolTable = new \BambooHR\Guardrail\SymbolTable\JsonSymbolTable($this->getSymbolTableFile(), $this->getBasePath());
-			} else {
-				$this->symbolTable = new \BambooHR\Guardrail\SymbolTable\SqliteSymbolTable($this->getSymbolTableFile(), $this->getBasePath());
-			}
+			$this->symbolTable = new \BambooHR\Guardrail\SymbolTable\JsonSymbolTable($this->getSymbolTableFile(), $this->getBasePath());
 		} else {
 			$this->forceIndex = true;
 			$this->symbolTable = new \BambooHR\Guardrail\SymbolTable\InMemorySymbolTable($this->getBasePath());
@@ -308,13 +304,6 @@ private function parseArgv(array $argv) {
 				case '-i':
 					$this->forceIndex = true;
 					break;
-				case '-s':
-					$this->preferredTable = self::SQLITE_SYMBOL_TABLE;
-					if (!extension_loaded("pdo_sqlite") || !extension_loaded("sqlite3")) {
-						echo "Guardrail requires the PDO Sqlite extension, which is not loaded.\n";
-						throw new InvalidConfigException();
-					}
-					break;
 				case '-m':
 					$this->preferredTable = self::MEMORY_SYMBOL_TABLE;
 					break;
@@ -502,8 +491,7 @@ public function getOutputFormat() {
 	 * @return string
 	 */
 	private function getSymbolTableFile() {
-		return $this->basePath . "/" . $this->symbolTableFile .
-			($this->preferredTable == self::SQLITE_SYMBOL_TABLE ? ".sqlite3" : ".json");
+		return $this->basePath . "/" . $this->symbolTableFile . ".json";
 	}
 
 	/**

diff --git a/src/Phases/IndexingPhase.php b/src/Phases/IndexingPhase.php
@@ -67,15 +67,20 @@ function __construct(Config $config) {
 	 *
 	 * @return \Generator
 	 */
-	private function getFileList(Config $config, \Iterator $it2, $stubs = false) {
+	private function getFileList(Config $config, array $paths, $stubs = false) {
 		$baseDir = $config->getBasePath();
 		$configArr = $config->getConfigArray();
-		foreach ($it2 as $filePath) {
-			if (preg_match('/\\.(php|inc)$/', $filePath) && is_file($filePath)) {
-				if (!$stubs && isset($configArr['ignore']) && is_array($configArr['ignore']) && Util::matchesGlobs($baseDir, $filePath, $configArr['ignore'])) {
-					continue;
+
+		foreach($paths as $path) {
+			$tmpDirectory = Util::fullDirectoryPath($baseDir, $path);
+			$generator = DirectoryLister::getGenerator($tmpDirectory);
+			foreach ($generator as $filePath) {
+				if (preg_match('/\\.(php|inc)$/', $filePath) && is_file($filePath)) {
+					if (!$stubs && isset($configArr['ignore']) && is_array($configArr['ignore']) && Util::matchesGlobs($baseDir, $filePath, $configArr['ignore'])) {
+						continue;
+					}
+					yield $filePath;
 				}
-				yield $filePath;
 			}
 		}
 	}
@@ -128,18 +133,18 @@ public function indexTraitClasses(SymbolTable $symbolTable, OutputInterface $out
 
 
 	/**
-	 * @param int    $fileNumber -
+	 * @param int    $processNumber -
 	 * @param Config $config     -
 	 * @return resource The client socket that the server should communicate with.
 	 */
-	function createIndexingChild($fileNumber, Config $config) {
+	function createIndexingChild($processNumber, Config $config) {
 		return $this->processManager->createChild(
 			// This closure represents the child process.  The value it returns
 			// will be the exit code of the child process.
-			function($socket) use($config, $fileNumber) {
+			function($socket) use($config, $processNumber) {
 				$table = $config->getSymbolTable();
 				if ($table instanceof PersistantSymbolTable) {
-					$table->connect( $fileNumber + 1 );
+					$table->connect($processNumber + 1 );
 				}
 				$buffer = new SocketBuffer();
 				while (1) {
@@ -154,7 +159,7 @@ function($socket) use($config, $fileNumber) {
 						} else {
 							list(, $file) = explode(' ', trim($receive));
 							$size = $this->indexFile($config, $file);
-							socket_write($socket, "INDEXED $size $file\n");
+							socket_write($socket, "INDEXED $size $file ".($processNumber+1)."\n");
 						}
 					}
 				}
@@ -178,7 +183,8 @@ function indexList(Config $config, OutputInterface $output, $itr) {
 		$bytes = 0.0;
 		// Fire up our child processes and give them each a file to index.
 		for ($fileNumber = 0; $fileNumber < $config->getProcessCount() && $itr->valid(); ++$fileNumber, $itr->next()) {
-			$child = $this->createIndexingChild($fileNumber, $config);
+			$processNumber = $fileNumber;
+			$child = $this->createIndexingChild($processNumber, $config);
 			socket_write($child, "INDEX " . $itr->current() . "\n");
 			$output->output(".", sprintf("%d - %s", $fileNumber, $itr->current()));
 		}
@@ -192,11 +198,12 @@ function ($socket, $msg) use (&$itr, &$fileNumber, &$bytes, $output, $start) {
 				list($message, $details) = explode(' ', $msg, 2);
 
 				if ($message == 'INDEXED') {
+					[$size, $fileName, $childProcessNumber] = explode(' ', $details);
+					$bytes += $size;
+					$output->output(".", sprintf("%d - %s ($childProcessNumber)", ++$fileNumber, $fileName));
+
 					if ($itr->valid()) {
-						list($size,) = explode(' ', $details);
-						$bytes += $size;
-						$output->output(".", sprintf("%d - %s", ++$fileNumber, $itr->current()));
-						socket_write($socket, "INDEX " . $itr->current() . "\n");
+						socket_write($socket, "INDEX " . $itr->current() ."\n");
 						$itr->next();
 					} else {
 						socket_write($socket, "DONE\n");
@@ -233,12 +240,7 @@ public function run(Config $config, OutputInterface $output) {
 		}
 		$output->outputVerbose("Index directories are valid: Indexing starting.");
 
-		foreach ($indexPaths as $path) {
-			$tmpDirectory = Util::fullDirectoryPath($baseDirectory, $path);
-			$output->outputVerbose("Indexing Directory: " . $tmpDirectory);
-			$generator = DirectoryLister::getGenerator($tmpDirectory);
-			$this->indexList($config, $output, $this->getFileList($config, $generator) );
-		}
+		$this->indexList($config, $output, $this->getFileList($config, $indexPaths) );
 
 		$table = $config->getSymbolTable();
 		if ($table instanceof PersistantSymbolTable) {

diff --git a/src/SymbolTable/JsonSymbolTable.php b/src/SymbolTable/JsonSymbolTable.php
@@ -30,7 +30,13 @@
 class JsonSymbolTable extends SymbolTable implements PersistantSymbolTable {
 
 	/** @var array [$type][$name] = ['has_trait'=>,'data'=>,'file'=>] */
-	private $index = [];
+	private $index = [
+		SymbolTable::TYPE_CLASS => [],
+		SymbolTable::TYPE_FUNCTION => [],
+		SymbolTable::TYPE_INTERFACE => [],
+		SymbolTable::TYPE_TRAIT => [],
+		SymbolTable::TYPE_DEFINE => []
+	];
 
 	private $processNumber = 0;
 
@@ -55,7 +61,8 @@ public function __construct($fileName, $basePath) {
 	 */
 	public function disconnect() {
 		$fileName = $this->fileName . ($this->processNumber ? '.' . $this->processNumber : '');
-		file_put_contents($fileName, json_encode($this->index));
+		$str=json_encode($this->index,JSON_THROW_ON_ERROR|JSON_INVALID_UTF8_SUBSTITUTE);
+		file_put_contents($fileName, $str) ;
 	}
 
 	/**
@@ -110,7 +117,8 @@ function flushInserts() {
 	function indexTable($processCount) {
 		for ($index = 1; $index <= $processCount; ++$index) {
 			$fileName = $this->fileName . '.' . $index;
-			$arr = json_decode(file_get_contents($fileName), true);
+			$content = file_get_contents($fileName);
+			$arr = json_decode($content, true);
 			foreach ($arr as $type => $arr2) {
 				if (!isset($this->index[$type])) {
 					$this->index[$type] = [];