[4.0] Removing artificial sharding from com_finder (#20185)

* Removing artificial sharding from com_finder * Fixing codestyle * the update sql for postgresql * Reverting changes to com_finder router
joomla · Jun 11, 2018 · 5613476 · 5613476
1 parent fcdfa20
commit 5613476
Show file tree

Hide file tree

Showing 16 changed files with 489 additions and 1,997 deletions.
diff --git a/administrator/components/com_admin/sql/updates/mysql/4.0.0-2018-04-14.sql b/administrator/components/com_admin/sql/updates/mysql/4.0.0-2018-04-14.sql
@@ -0,0 +1,45 @@
+CREATE TABLE `#__finder_links_terms` (
+	`link_id` INT(10) UNSIGNED NOT NULL,
+	`term_id` INT(10) UNSIGNED NOT NULL,
+	`weight` FLOAT UNSIGNED NOT NULL,
+	PRIMARY KEY (`link_id`, `term_id`),
+	INDEX `idx_term_weight` (`term_id`, `weight`),
+	INDEX `idx_link_term_weight` (`link_id`, `term_id`, `weight`)
+)
+COLLATE='utf8mb4_general_ci'
+ENGINE=InnoDB
+;
+DROP TABLE #__finder_links_terms0;
+DROP TABLE #__finder_links_terms1;
+DROP TABLE #__finder_links_terms2;
+DROP TABLE #__finder_links_terms3;
+DROP TABLE #__finder_links_terms4;
+DROP TABLE #__finder_links_terms5;
+DROP TABLE #__finder_links_terms6;
+DROP TABLE #__finder_links_terms7;
+DROP TABLE #__finder_links_terms8;
+DROP TABLE #__finder_links_terms9;
+DROP TABLE #__finder_links_termsa;
+DROP TABLE #__finder_links_termsb;
+DROP TABLE #__finder_links_termsc;
+DROP TABLE #__finder_links_termsd;
+DROP TABLE #__finder_links_termse;
+DROP TABLE #__finder_links_termsf;
+
+ALTER TABLE `#__finder_terms`
+	CHANGE COLUMN `language` `language` CHAR(7) NOT NULL DEFAULT '' AFTER `links`;
+
+ALTER TABLE `#__finder_terms_common`
+	CHANGE COLUMN `language` `language` CHAR(7) NOT NULL DEFAULT '' AFTER `term`;
+
+ALTER TABLE `#__finder_tokens`
+	CHANGE COLUMN `language` `language` CHAR(7) NOT NULL DEFAULT '' AFTER `context`;
+
+ALTER TABLE `#__finder_tokens_aggregate`
+	DROP COLUMN `map_suffix`;
+
+ALTER TABLE `#__finder_tokens_aggregate`
+	CHANGE COLUMN `language` `language` CHAR(7) NOT NULL DEFAULT '' AFTER `total_weight`;
+
+ALTER TABLE `#__finder_links`
+	CHANGE COLUMN `language` `language` CHAR(7) NOT NULL DEFAULT '' AFTER `access`;
diff --git a/administrator/components/com_admin/sql/updates/postgresql/4.0.0-2018-04-14.sql b/administrator/components/com_admin/sql/updates/postgresql/4.0.0-2018-04-14.sql
@@ -0,0 +1,48 @@
+CREATE TABLE "#__finder_links_terms" (
+	"link_id" bigint NOT NULL,
+	"term_id" bigint NOT NULL,
+	"weight" REAL NOT NULL,
+	PRIMARY KEY ("link_id", "term_id")
+);
+CREATE INDEX "idx_term_weight" ("term_id", "weight");
+CREATE INDEX "idx_link_term_weight" ("link_id", "term_id", "weight");
+
+DROP TABLE "#__finder_links_terms0" CASCADE;
+DROP TABLE "#__finder_links_terms1" CASCADE;
+DROP TABLE "#__finder_links_terms2" CASCADE;
+DROP TABLE "#__finder_links_terms3" CASCADE;
+DROP TABLE "#__finder_links_terms4" CASCADE;
+DROP TABLE "#__finder_links_terms5" CASCADE;
+DROP TABLE "#__finder_links_terms6" CASCADE;
+DROP TABLE "#__finder_links_terms7" CASCADE;
+DROP TABLE "#__finder_links_terms8" CASCADE;
+DROP TABLE "#__finder_links_terms9" CASCADE;
+DROP TABLE "#__finder_links_termsa" CASCADE;
+DROP TABLE "#__finder_links_termsb" CASCADE;
+DROP TABLE "#__finder_links_termsc" CASCADE;
+DROP TABLE "#__finder_links_termsd" CASCADE;
+DROP TABLE "#__finder_links_termse" CASCADE;
+DROP TABLE "#__finder_links_termsf" CASCADE;
+
+ALTER TABLE "#__finder_terms"
+	ALTER COLUMN "language" TYPE CHAR(7),
+	ALTER COLUMN "language" SET DEFAULT '';
+
+ALTER TABLE "#__finder_common"
+	ALTER COLUMN "language" TYPE CHAR(7),
+	ALTER COLUMN "language" SET DEFAULT '';
+
+ALTER TABLE "#__finder_tokens"
+	ALTER COLUMN "language" TYPE CHAR(7),
+	ALTER COLUMN "language" SET DEFAULT '';
+
+ALTER TABLE "#__finder_tokens_aggregate"
+	ALTER COLUMN "language" TYPE CHAR(7),
+	ALTER COLUMN "language" SET DEFAULT '';
+
+ALTER TABLE "#__finder_tokens_aggregate"
+	DROP COLUMN "map_suffix";
+
+ALTER TABLE "#__finder_links"
+	ALTER COLUMN "language" TYPE CHAR(7),
+	ALTER COLUMN "language" SET DEFAULT '';
diff --git a/administrator/components/com_finder/Model/IndexModel.php b/administrator/components/com_finder/Model/IndexModel.php
@@ -342,13 +342,7 @@ public function purge()
 		$db->truncateTable('#__finder_links');
 
 		// Truncate the links terms tables.
-		for ($i = 0; $i <= 15; $i++)
-		{
-			// Get the mapping table suffix.
-			$suffix = dechex($i);
-
-			$db->truncateTable('#__finder_links_terms' . $suffix);
-		}
+		$db->truncateTable('#__finder_links_terms');
 
 		// Truncate the terms table.
 		$db->truncateTable('#__finder_terms');

diff --git a/administrator/components/com_finder/helpers/indexer/driver/mysql.php b/administrator/components/com_finder/helpers/indexer/driver/mysql.php
@@ -78,15 +78,12 @@ public function index($item, $format = 'html')
 		 */
 		if (!$isNew)
 		{
-			for ($i = 0; $i <= 15; $i++)
-			{
-				// Flush the maps for the link.
-				$query->clear()
-					->delete($db->quoteName('#__finder_links_terms' . dechex($i)))
-					->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
-				$db->setQuery($query);
-				$db->execute();
-			}
+			// Flush the maps for the link.
+			$query->clear()
+				->delete($db->quoteName('#__finder_links_terms'))
+				->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
+			$db->setQuery($query);
+			$db->execute();
 
 			// Remove the taxonomy maps.
 			FinderIndexerTaxonomy::removeMaps($linkId);
@@ -288,8 +285,7 @@ public function index($item, $format = 'html')
 		 */
 		$query = 'INSERT INTO ' . $db->quoteName('#__finder_tokens_aggregate') .
 			' (' . $db->quoteName('term_id') .
-			', ' . $db->quoteName('map_suffix') .
-				', ' . $db->quoteName('term') .
+			', ' . $db->quoteName('term') .
 			', ' . $db->quoteName('stem') .
 			', ' . $db->quoteName('common') .
 			', ' . $db->quoteName('phrase') .
@@ -299,7 +295,7 @@ public function index($item, $format = 'html')
 			', ' . $db->quoteName('total_weight') .
 				', ' . $db->quoteName('language') . ')' .
 			' SELECT' .
-			' COALESCE(t.term_id, 0), \'\', t1.term, t1.stem, t1.common, t1.phrase, t1.weight, t1.context,' .
+			' COALESCE(t.term_id, 0), t1.term, t1.stem, t1.common, t1.phrase, t1.weight, t1.context,' .
 			' ROUND( t1.weight * COUNT( t2.term ) * %F, 8 ) AS context_weight, 0, t1.language' .
 			' FROM (' .
 			'   SELECT DISTINCT t1.term, t1.stem, t1.common, t1.phrase, t1.weight, t1.context, t1.language' .
@@ -377,50 +373,25 @@ public function index($item, $format = 'html')
 		// Mark afterTerms in the profiler.
 		static::$profiler ? static::$profiler->mark('afterTerms') : null;
 
-		/*
-		 * Before we can insert all of the mapping rows, we have to figure out
-		 * which mapping table the rows need to be inserted into. The mapping
-		 * table for each term is based on the first character of the md5 of
-		 * the first character of the term. In php, it would be expressed as
-		 * substr(md5(substr($token, 0, 1)), 0, 1)
-		 */
-		$query->clear()
-			->update($db->quoteName('#__finder_tokens_aggregate'))
-			->set($db->quoteName('map_suffix') . ' = SUBSTR(MD5(SUBSTR(' . $db->quoteName('term') . ', 1, 1)), 1, 1)');
-		$db->setQuery($query);
-		$db->execute();
-
 		/*
 		 * At this point, the aggregate table contains a record for each
 		 * term in each context. So, we're going to pull down all of that
 		 * data while grouping the records by term and add all of the
 		 * sub-totals together to arrive at the final total for each token for
-		 * this link. Then, we insert all of that data into the appropriate
-		 * mapping table.
+		 * this link. Then, we insert all of that data into the mapping table.
 		 */
-		for ($i = 0; $i <= 15; $i++)
-		{
-			// Get the mapping table suffix.
-			$suffix = dechex($i);
-
-			/*
-			 * We have to run this query 16 times, one for each link => term
-			 * mapping table.
-			 */
-			$db->setQuery(
-				'INSERT INTO ' . $db->quoteName('#__finder_links_terms' . $suffix) .
-				' (' . $db->quoteName('link_id') .
-				', ' . $db->quoteName('term_id') .
-				', ' . $db->quoteName('weight') . ')' .
-				' SELECT ' . (int) $linkId . ', ' . $db->quoteName('term_id') . ',' .
-				' ROUND(SUM(' . $db->quoteName('context_weight') . '), 8)' .
-				' FROM ' . $db->quoteName('#__finder_tokens_aggregate') .
-				' WHERE ' . $db->quoteName('map_suffix') . ' = ' . $db->quote($suffix) .
-				' GROUP BY ' . $db->quoteName('term') . ', ' . $db->quoteName('term_id') .
-				' ORDER BY ' . $db->quoteName('term') . ' DESC'
-			);
-			$db->execute();
-		}
+		$db->setQuery(
+			'INSERT INTO ' . $db->quoteName('#__finder_links_terms') .
+			' (' . $db->quoteName('link_id') .
+			', ' . $db->quoteName('term_id') .
+			', ' . $db->quoteName('weight') . ')' .
+			' SELECT ' . (int) $linkId . ', ' . $db->quoteName('term_id') . ',' .
+			' ROUND(SUM(' . $db->quoteName('context_weight') . '), 8)' .
+			' FROM ' . $db->quoteName('#__finder_tokens_aggregate') .
+			' GROUP BY ' . $db->quoteName('term') . ', ' . $db->quoteName('term_id') .
+			' ORDER BY ' . $db->quoteName('term') . ' DESC'
+		);
+		$db->execute();
 
 		// Mark afterMapping in the profiler.
 		static::$profiler ? static::$profiler->mark('afterMapping') : null;
@@ -476,12 +447,9 @@ public function optimize()
 		$db->setQuery('OPTIMIZE TABLE ' . $db->quoteName('#__finder_links'));
 		$db->execute();
 
-		for ($i = 0; $i <= 15; $i++)
-		{
-			// Optimize the terms mapping table.
-			$db->setQuery('OPTIMIZE TABLE ' . $db->quoteName('#__finder_links_terms' . dechex($i)));
-			$db->execute();
-		}
+		// Optimize the terms mapping table.
+		$db->setQuery('OPTIMIZE TABLE ' . $db->quoteName('#__finder_links_terms'));
+		$db->execute();
 
 		// Optimize the filters table.
 		$db->setQuery('OPTIMIZE TABLE ' . $db->quoteName('#__finder_filters'));

diff --git a/administrator/components/com_finder/helpers/indexer/driver/postgresql.php b/administrator/components/com_finder/helpers/indexer/driver/postgresql.php
@@ -70,15 +70,12 @@ public function index($item, $format = 'html')
 		 */
 		if (!$isNew)
 		{
-			for ($i = 0; $i <= 15; $i++)
-			{
-				// Flush the maps for the link.
-				$query->clear()
-					->delete($db->quoteName('#__finder_links_terms' . dechex($i)))
-					->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
-				$db->setQuery($query);
-				$db->execute();
-			}
+			// Flush the maps for the link.
+			$query->clear()
+				->delete($db->quoteName('#__finder_links_terms'))
+				->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
+			$db->setQuery($query);
+			$db->execute();
 
 			// Remove the taxonomy maps.
 			FinderIndexerTaxonomy::removeMaps($linkId);
@@ -378,19 +375,6 @@ public function index($item, $format = 'html')
 		// Mark afterTerms in the profiler.
 		static::$profiler ? static::$profiler->mark('afterTerms') : null;
 
-		/*
-		 * Before we can insert all of the mapping rows, we have to figure out
-		 * which mapping table the rows need to be inserted into. The mapping
-		 * table for each term is based on the first character of the md5 of
-		 * the first character of the term. In php, it would be expressed as
-		 * substr(md5(substr($token, 0, 1)), 0, 1)
-		 */
-		$query->clear()
-			->update($db->quoteName('#__finder_tokens_aggregate'))
-			->set($db->quoteName('map_suffix') . ' = SUBSTR(MD5(SUBSTR(' . $db->quoteName('term') . ', 1, 1)), 1, 1)');
-		$db->setQuery($query);
-		$db->execute();
-
 		/*
 		 * At this point, the aggregate table contains a record for each
 		 * term in each context. So, we're going to pull down all of that
@@ -399,29 +383,18 @@ public function index($item, $format = 'html')
 		 * this link. Then, we insert all of that data into the appropriate
 		 * mapping table.
 		 */
-		for ($i = 0; $i <= 15; $i++)
-		{
-			// Get the mapping table suffix.
-			$suffix = dechex($i);
-
-			/*
-			 * We have to run this query 16 times, one for each link => term
-			 * mapping table.
-			 */
-			$db->setQuery(
-				'INSERT INTO ' . $db->quoteName('#__finder_links_terms' . $suffix) .
-				' (' . $db->quoteName('link_id') .
-				', ' . $db->quoteName('term_id') .
-				', ' . $db->quoteName('weight') . ')' .
-				' SELECT ' . (int) $linkId . ', ' . $db->quoteName('term_id') . ',' .
-				' ROUND(SUM(' . $db->quoteName('context_weight') . '), 8)' .
-				' FROM ' . $db->quoteName('#__finder_tokens_aggregate') .
-				' WHERE ' . $db->quoteName('map_suffix') . ' = ' . $db->quote($suffix) .
-				' GROUP BY ' . $db->quoteName('term') . ', ' . $db->quoteName('term_id') .
-				' ORDER BY ' . $db->quoteName('term') . ' DESC'
-			);
-			$db->execute();
-		}
+		$db->setQuery(
+			'INSERT INTO ' . $db->quoteName('#__finder_links_terms') .
+			' (' . $db->quoteName('link_id') .
+			', ' . $db->quoteName('term_id') .
+			', ' . $db->quoteName('weight') . ')' .
+			' SELECT ' . (int) $linkId . ', ' . $db->quoteName('term_id') . ',' .
+			' ROUND(SUM(' . $db->quoteName('context_weight') . '), 8)' .
+			' FROM ' . $db->quoteName('#__finder_tokens_aggregate') .
+			' GROUP BY ' . $db->quoteName('term') . ', ' . $db->quoteName('term_id') .
+			' ORDER BY ' . $db->quoteName('term') . ' DESC'
+		);
+		$db->execute();
 
 		// Mark afterMapping in the profiler.
 		static::$profiler ? static::$profiler->mark('afterMapping') : null;
@@ -479,14 +452,11 @@ public function optimize()
 		$db->setQuery('REINDEX TABLE ' . $db->quoteName('#__finder_links'));
 		$db->execute();
 
-		for ($i = 0; $i <= 15; $i++)
-		{
-			// Optimize the terms mapping table.
-			$db->setQuery('VACUUM ' . $db->quoteName('#__finder_links_terms' . dechex($i)));
-			$db->execute();
-			$db->setQuery('REINDEX TABLE ' . $db->quoteName('#__finder_links_terms' . dechex($i)));
-			$db->execute();
-		}
+		// Optimize the terms mapping table.
+		$db->setQuery('VACUUM ' . $db->quoteName('#__finder_links_terms'));
+		$db->execute();
+		$db->setQuery('REINDEX TABLE ' . $db->quoteName('#__finder_links_terms'));
+		$db->execute();
 
 		// Optimize the filters table.
 		$db->setQuery('REINDEX TABLE ' . $db->quoteName('#__finder_filters'));

diff --git a/administrator/components/com_finder/helpers/indexer/indexer.php b/administrator/components/com_finder/helpers/indexer/indexer.php
@@ -295,23 +295,19 @@ public function remove($linkId)
 		$db    = $this->db;
 		$query = $db->getQuery(true);
 
-		// Update the link counts and remove the mapping records.
-		for ($i = 0; $i <= 15; $i++)
-		{
-			// Update the link counts for the terms.
-			$query->clear()
-				->update($db->quoteName('#__finder_terms', 't'))
-				->join('INNER', $db->quoteName('#__finder_links_terms' . dechex($i), 'm') . ' ON m.term_id = t.term_id')
-				->set('t.links = t.links - 1')
-				->where($db->quoteName('m.link_id') . ' = ' . (int) $linkId);
-			$db->setQuery($query)->execute();
+		// Update the link counts for the terms.
+		$query->clear()
+			->update($db->quoteName('#__finder_terms', 't'))
+			->join('INNER', $db->quoteName('#__finder_links_terms', 'm') . ' ON m.term_id = t.term_id')
+			->set('t.links = t.links - 1')
+			->where($db->quoteName('m.link_id') . ' = ' . (int) $linkId);
+		$db->setQuery($query)->execute();
 
-			// Remove all records from the mapping tables.
-			$query->clear()
-				->delete($db->quoteName('#__finder_links_terms' . dechex($i)))
-				->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
-			$db->setQuery($query)->execute();
-		}
+		// Remove all records from the mapping tables.
+		$query->clear()
+			->delete($db->quoteName('#__finder_links_terms'))
+			->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
+		$db->setQuery($query)->execute();
 
 		// Delete all orphaned terms.
 		$query->clear()