Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[4.0] Removing artificial sharding from com_finder #20185

Merged
merged 7 commits into from
Jun 11, 2018
Merged

[4.0] Removing artificial sharding from com_finder #20185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d437f07
Removing artificial sharding from com_finder
Hackwar Apr 17, 2018
8d12204
Fixing codestyle
Hackwar Apr 17, 2018
ad345a2
Merge branch '4.0-dev' into j4finder_sharding
Hackwar Apr 17, 2018
3763187
the update sql for postgresql
alikon Apr 21, 2018
7dd2bea
Merge pull request #22 from alikon/patch-104
Hackwar Apr 21, 2018
455fc19
Reverting changes to com_finder router
Hackwar Jun 5, 2018
c80b975
Merge branch '4.0-dev' of https://github.com/joomla/joomla-cms into j…
Hackwar Jun 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions administrator/components/com_admin/sql/updates/mysql/4.0.0-2018-04-14.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
CREATE TABLE `#__finder_links_terms` (
`link_id` INT(10) UNSIGNED NOT NULL,
`term_id` INT(10) UNSIGNED NOT NULL,
`weight` FLOAT UNSIGNED NOT NULL,
PRIMARY KEY (`link_id`, `term_id`),
INDEX `idx_term_weight` (`term_id`, `weight`),
INDEX `idx_link_term_weight` (`link_id`, `term_id`, `weight`)
)
COLLATE='utf8mb4_general_ci'
ENGINE=InnoDB
;
DROP TABLE #__finder_links_terms0;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tables are not dropped and I don't know why.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did this operation manually, and work ok

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, but not via Extensions > Manage > Database > Fix

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Iirc you can't do a drop that way

DROP TABLE #__finder_links_terms1;
DROP TABLE #__finder_links_terms2;
DROP TABLE #__finder_links_terms3;
DROP TABLE #__finder_links_terms4;
DROP TABLE #__finder_links_terms5;
DROP TABLE #__finder_links_terms6;
DROP TABLE #__finder_links_terms7;
DROP TABLE #__finder_links_terms8;
DROP TABLE #__finder_links_terms9;
DROP TABLE #__finder_links_termsa;
DROP TABLE #__finder_links_termsb;
DROP TABLE #__finder_links_termsc;
DROP TABLE #__finder_links_termsd;
DROP TABLE #__finder_links_termse;
DROP TABLE #__finder_links_termsf;

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed CHANGE COLUMN to CHANGE, dropped AFTER ... and be on same line to make it work.

ALTER TABLE `#__finder_terms` CHANGE `language` `language` CHAR(7) NOT NULL DEFAULT '';
ALTER TABLE `#__finder_terms_common` CHANGE `language` `language` CHAR(7) NOT NULL DEFAULT '';
ALTER TABLE `#__finder_tokens` CHANGE `language` `language` CHAR(7) NOT NULL DEFAULT '';
ALTER TABLE `#__finder_tokens_aggregate` DROP COLUMN `map_suffix`;
ALTER TABLE `#__finder_tokens_aggregate` CHANGE `language` `language` CHAR(7) NOT NULL DEFAULT '';
ALTER TABLE `#__finder_links` CHANGE `language` `language` CHAR(7) NOT NULL DEFAULT '';

ALTER TABLE `#__finder_terms`
CHANGE COLUMN `language` `language` CHAR(7) NOT NULL DEFAULT '' AFTER `links`;

ALTER TABLE `#__finder_terms_common`
CHANGE COLUMN `language` `language` CHAR(7) NOT NULL DEFAULT '' AFTER `term`;

ALTER TABLE `#__finder_tokens`
CHANGE COLUMN `language` `language` CHAR(7) NOT NULL DEFAULT '' AFTER `context`;

ALTER TABLE `#__finder_tokens_aggregate`
DROP COLUMN `map_suffix`;

ALTER TABLE `#__finder_tokens_aggregate`
CHANGE COLUMN `language` `language` CHAR(7) NOT NULL DEFAULT '' AFTER `total_weight`;

ALTER TABLE `#__finder_links`
CHANGE COLUMN `language` `language` CHAR(7) NOT NULL DEFAULT '' AFTER `access`;
48 changes: 48 additions & 0 deletions administrator/components/com_admin/sql/updates/postgresql/4.0.0-2018-04-14.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
CREATE TABLE "#__finder_links_terms" (
"link_id" bigint NOT NULL,
"term_id" bigint NOT NULL,
"weight" REAL NOT NULL,
PRIMARY KEY ("link_id", "term_id")
);
CREATE INDEX "idx_term_weight" ("term_id", "weight");
CREATE INDEX "idx_link_term_weight" ("link_id", "term_id", "weight");

DROP TABLE "#__finder_links_terms0" CASCADE;
DROP TABLE "#__finder_links_terms1" CASCADE;
DROP TABLE "#__finder_links_terms2" CASCADE;
DROP TABLE "#__finder_links_terms3" CASCADE;
DROP TABLE "#__finder_links_terms4" CASCADE;
DROP TABLE "#__finder_links_terms5" CASCADE;
DROP TABLE "#__finder_links_terms6" CASCADE;
DROP TABLE "#__finder_links_terms7" CASCADE;
DROP TABLE "#__finder_links_terms8" CASCADE;
DROP TABLE "#__finder_links_terms9" CASCADE;
DROP TABLE "#__finder_links_termsa" CASCADE;
DROP TABLE "#__finder_links_termsb" CASCADE;
DROP TABLE "#__finder_links_termsc" CASCADE;
DROP TABLE "#__finder_links_termsd" CASCADE;
DROP TABLE "#__finder_links_termse" CASCADE;
DROP TABLE "#__finder_links_termsf" CASCADE;

ALTER TABLE "#__finder_terms"
ALTER COLUMN "language" TYPE CHAR(7),
ALTER COLUMN "language" SET DEFAULT '';

ALTER TABLE "#__finder_common"
ALTER COLUMN "language" TYPE CHAR(7),
ALTER COLUMN "language" SET DEFAULT '';

ALTER TABLE "#__finder_tokens"
ALTER COLUMN "language" TYPE CHAR(7),
ALTER COLUMN "language" SET DEFAULT '';

ALTER TABLE "#__finder_tokens_aggregate"
ALTER COLUMN "language" TYPE CHAR(7),
ALTER COLUMN "language" SET DEFAULT '';

ALTER TABLE "#__finder_tokens_aggregate"
DROP COLUMN "map_suffix";

ALTER TABLE "#__finder_links"
ALTER COLUMN "language" TYPE CHAR(7),
ALTER COLUMN "language" SET DEFAULT '';
8 changes: 1 addition & 7 deletions administrator/components/com_finder/Model/IndexModel.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,13 +342,7 @@ public function purge()
$db->truncateTable('#__finder_links');

// Truncate the links terms tables.
for ($i = 0; $i <= 15; $i++)
{
// Get the mapping table suffix.
$suffix = dechex($i);

$db->truncateTable('#__finder_links_terms' . $suffix);
}
$db->truncateTable('#__finder_links_terms');

// Truncate the terms table.
$db->truncateTable('#__finder_terms');
Expand Down
80 changes: 24 additions & 56 deletions administrator/components/com_finder/helpers/indexer/driver/mysql.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,15 +78,12 @@ public function index($item, $format = 'html')
*/
if (!$isNew)
{
for ($i = 0; $i <= 15; $i++)
{
// Flush the maps for the link.
$query->clear()
->delete($db->quoteName('#__finder_links_terms' . dechex($i)))
->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
$db->setQuery($query);
$db->execute();
}
// Flush the maps for the link.
$query->clear()
->delete($db->quoteName('#__finder_links_terms'))
->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
$db->setQuery($query);
$db->execute();

// Remove the taxonomy maps.
FinderIndexerTaxonomy::removeMaps($linkId);
Expand Down Expand Up @@ -288,8 +285,7 @@ public function index($item, $format = 'html')
*/
$query = 'INSERT INTO ' . $db->quoteName('#__finder_tokens_aggregate') .
' (' . $db->quoteName('term_id') .
', ' . $db->quoteName('map_suffix') .
', ' . $db->quoteName('term') .
', ' . $db->quoteName('term') .
', ' . $db->quoteName('stem') .
', ' . $db->quoteName('common') .
', ' . $db->quoteName('phrase') .
Expand All @@ -299,7 +295,7 @@ public function index($item, $format = 'html')
', ' . $db->quoteName('total_weight') .
', ' . $db->quoteName('language') . ')' .
' SELECT' .
' COALESCE(t.term_id, 0), \'\', t1.term, t1.stem, t1.common, t1.phrase, t1.weight, t1.context,' .
' COALESCE(t.term_id, 0), t1.term, t1.stem, t1.common, t1.phrase, t1.weight, t1.context,' .
' ROUND( t1.weight * COUNT( t2.term ) * %F, 8 ) AS context_weight, 0, t1.language' .
' FROM (' .
' SELECT DISTINCT t1.term, t1.stem, t1.common, t1.phrase, t1.weight, t1.context, t1.language' .
Expand Down Expand Up @@ -377,50 +373,25 @@ public function index($item, $format = 'html')
// Mark afterTerms in the profiler.
static::$profiler ? static::$profiler->mark('afterTerms') : null;

/*
* Before we can insert all of the mapping rows, we have to figure out
* which mapping table the rows need to be inserted into. The mapping
* table for each term is based on the first character of the md5 of
* the first character of the term. In php, it would be expressed as
* substr(md5(substr($token, 0, 1)), 0, 1)
*/
$query->clear()
->update($db->quoteName('#__finder_tokens_aggregate'))
->set($db->quoteName('map_suffix') . ' = SUBSTR(MD5(SUBSTR(' . $db->quoteName('term') . ', 1, 1)), 1, 1)');
$db->setQuery($query);
$db->execute();

/*
* At this point, the aggregate table contains a record for each
* term in each context. So, we're going to pull down all of that
* data while grouping the records by term and add all of the
* sub-totals together to arrive at the final total for each token for
* this link. Then, we insert all of that data into the appropriate
* mapping table.
* this link. Then, we insert all of that data into the mapping table.
*/
for ($i = 0; $i <= 15; $i++)
{
// Get the mapping table suffix.
$suffix = dechex($i);

/*
* We have to run this query 16 times, one for each link => term
* mapping table.
*/
$db->setQuery(
'INSERT INTO ' . $db->quoteName('#__finder_links_terms' . $suffix) .
' (' . $db->quoteName('link_id') .
', ' . $db->quoteName('term_id') .
', ' . $db->quoteName('weight') . ')' .
' SELECT ' . (int) $linkId . ', ' . $db->quoteName('term_id') . ',' .
' ROUND(SUM(' . $db->quoteName('context_weight') . '), 8)' .
' FROM ' . $db->quoteName('#__finder_tokens_aggregate') .
' WHERE ' . $db->quoteName('map_suffix') . ' = ' . $db->quote($suffix) .
' GROUP BY ' . $db->quoteName('term') . ', ' . $db->quoteName('term_id') .
' ORDER BY ' . $db->quoteName('term') . ' DESC'
);
$db->execute();
}
$db->setQuery(
'INSERT INTO ' . $db->quoteName('#__finder_links_terms') .
' (' . $db->quoteName('link_id') .
', ' . $db->quoteName('term_id') .
', ' . $db->quoteName('weight') . ')' .
' SELECT ' . (int) $linkId . ', ' . $db->quoteName('term_id') . ',' .
' ROUND(SUM(' . $db->quoteName('context_weight') . '), 8)' .
' FROM ' . $db->quoteName('#__finder_tokens_aggregate') .
' GROUP BY ' . $db->quoteName('term') . ', ' . $db->quoteName('term_id') .
' ORDER BY ' . $db->quoteName('term') . ' DESC'
);
$db->execute();

// Mark afterMapping in the profiler.
static::$profiler ? static::$profiler->mark('afterMapping') : null;
Expand Down Expand Up @@ -476,12 +447,9 @@ public function optimize()
$db->setQuery('OPTIMIZE TABLE ' . $db->quoteName('#__finder_links'));
$db->execute();

for ($i = 0; $i <= 15; $i++)
{
// Optimize the terms mapping table.
$db->setQuery('OPTIMIZE TABLE ' . $db->quoteName('#__finder_links_terms' . dechex($i)));
$db->execute();
}
// Optimize the terms mapping table.
$db->setQuery('OPTIMIZE TABLE ' . $db->quoteName('#__finder_links_terms'));
$db->execute();

// Optimize the filters table.
$db->setQuery('OPTIMIZE TABLE ' . $db->quoteName('#__finder_filters'));
Expand Down
76 changes: 23 additions & 53 deletions administrator/components/com_finder/helpers/indexer/driver/postgresql.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,12 @@ public function index($item, $format = 'html')
*/
if (!$isNew)
{
for ($i = 0; $i <= 15; $i++)
{
// Flush the maps for the link.
$query->clear()
->delete($db->quoteName('#__finder_links_terms' . dechex($i)))
->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
$db->setQuery($query);
$db->execute();
}
// Flush the maps for the link.
$query->clear()
->delete($db->quoteName('#__finder_links_terms'))
->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
$db->setQuery($query);
$db->execute();

// Remove the taxonomy maps.
FinderIndexerTaxonomy::removeMaps($linkId);
Expand Down Expand Up @@ -378,19 +375,6 @@ public function index($item, $format = 'html')
// Mark afterTerms in the profiler.
static::$profiler ? static::$profiler->mark('afterTerms') : null;

/*
* Before we can insert all of the mapping rows, we have to figure out
* which mapping table the rows need to be inserted into. The mapping
* table for each term is based on the first character of the md5 of
* the first character of the term. In php, it would be expressed as
* substr(md5(substr($token, 0, 1)), 0, 1)
*/
$query->clear()
->update($db->quoteName('#__finder_tokens_aggregate'))
->set($db->quoteName('map_suffix') . ' = SUBSTR(MD5(SUBSTR(' . $db->quoteName('term') . ', 1, 1)), 1, 1)');
$db->setQuery($query);
$db->execute();

/*
* At this point, the aggregate table contains a record for each
* term in each context. So, we're going to pull down all of that
Expand All @@ -399,29 +383,18 @@ public function index($item, $format = 'html')
* this link. Then, we insert all of that data into the appropriate
* mapping table.
*/
for ($i = 0; $i <= 15; $i++)
{
// Get the mapping table suffix.
$suffix = dechex($i);

/*
* We have to run this query 16 times, one for each link => term
* mapping table.
*/
$db->setQuery(
'INSERT INTO ' . $db->quoteName('#__finder_links_terms' . $suffix) .
' (' . $db->quoteName('link_id') .
', ' . $db->quoteName('term_id') .
', ' . $db->quoteName('weight') . ')' .
' SELECT ' . (int) $linkId . ', ' . $db->quoteName('term_id') . ',' .
' ROUND(SUM(' . $db->quoteName('context_weight') . '), 8)' .
' FROM ' . $db->quoteName('#__finder_tokens_aggregate') .
' WHERE ' . $db->quoteName('map_suffix') . ' = ' . $db->quote($suffix) .
' GROUP BY ' . $db->quoteName('term') . ', ' . $db->quoteName('term_id') .
' ORDER BY ' . $db->quoteName('term') . ' DESC'
);
$db->execute();
}
$db->setQuery(
'INSERT INTO ' . $db->quoteName('#__finder_links_terms') .
' (' . $db->quoteName('link_id') .
', ' . $db->quoteName('term_id') .
', ' . $db->quoteName('weight') . ')' .
' SELECT ' . (int) $linkId . ', ' . $db->quoteName('term_id') . ',' .
' ROUND(SUM(' . $db->quoteName('context_weight') . '), 8)' .
' FROM ' . $db->quoteName('#__finder_tokens_aggregate') .
' GROUP BY ' . $db->quoteName('term') . ', ' . $db->quoteName('term_id') .
' ORDER BY ' . $db->quoteName('term') . ' DESC'
);
$db->execute();

// Mark afterMapping in the profiler.
static::$profiler ? static::$profiler->mark('afterMapping') : null;
Expand Down Expand Up @@ -479,14 +452,11 @@ public function optimize()
$db->setQuery('REINDEX TABLE ' . $db->quoteName('#__finder_links'));
$db->execute();

for ($i = 0; $i <= 15; $i++)
{
// Optimize the terms mapping table.
$db->setQuery('VACUUM ' . $db->quoteName('#__finder_links_terms' . dechex($i)));
$db->execute();
$db->setQuery('REINDEX TABLE ' . $db->quoteName('#__finder_links_terms' . dechex($i)));
$db->execute();
}
// Optimize the terms mapping table.
$db->setQuery('VACUUM ' . $db->quoteName('#__finder_links_terms'));
$db->execute();
$db->setQuery('REINDEX TABLE ' . $db->quoteName('#__finder_links_terms'));
$db->execute();

// Optimize the filters table.
$db->setQuery('REINDEX TABLE ' . $db->quoteName('#__finder_filters'));
Expand Down
28 changes: 12 additions & 16 deletions administrator/components/com_finder/helpers/indexer/indexer.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,23 +295,19 @@ public function remove($linkId)
$db = $this->db;
$query = $db->getQuery(true);

// Update the link counts and remove the mapping records.
for ($i = 0; $i <= 15; $i++)
{
// Update the link counts for the terms.
$query->clear()
->update($db->quoteName('#__finder_terms', 't'))
->join('INNER', $db->quoteName('#__finder_links_terms' . dechex($i), 'm') . ' ON m.term_id = t.term_id')
->set('t.links = t.links - 1')
->where($db->quoteName('m.link_id') . ' = ' . (int) $linkId);
$db->setQuery($query)->execute();
// Update the link counts for the terms.
$query->clear()
->update($db->quoteName('#__finder_terms', 't'))
->join('INNER', $db->quoteName('#__finder_links_terms', 'm') . ' ON m.term_id = t.term_id')
->set('t.links = t.links - 1')
->where($db->quoteName('m.link_id') . ' = ' . (int) $linkId);
$db->setQuery($query)->execute();

// Remove all records from the mapping tables.
$query->clear()
->delete($db->quoteName('#__finder_links_terms' . dechex($i)))
->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
$db->setQuery($query)->execute();
}
// Remove all records from the mapping tables.
$query->clear()
->delete($db->quoteName('#__finder_links_terms'))
->where($db->quoteName('link_id') . ' = ' . (int) $linkId);
$db->setQuery($query)->execute();

// Delete all orphaned terms.
$query->clear()
Expand Down
Loading