Make it possible to define joins for log tables using `getWaysToJoinT…

…oOtherLogTables` (matomo-org#14062)

* Make it possible to define joins for log tables using getWaysToJoinToOtherLogTables

* Adds some tests for custom log table joins

* add missing log tables joined using getWaysToJoinToOtherLogTables

* automatically add log tables up the hierarchy

* code improvements

* Adds new ExampleLogTables plugin giving a showcase for custom log tables

* specifiy table name in userid archiver to fix query if custom log table joins on user_id column

* fix tests

* Adds log table that does only indirectly join with log_visit

* Allow defining joins on visit and action

* update ui files

core/Application/Kernel/PluginList.php

@@ -39,6 +39,7 @@ class PluginList
         'ExampleVisualization',
         'ExamplePluginTemplate',
         'ExampleTracker',
+        'ExampleLogTables',
         'ExampleReport',
         'MobileAppMeasurable',
         'Provider',

core/DataAccess/LogQueryBuilder/JoinGenerator.php

@@ -53,6 +53,19 @@ private function addMissingTablesNeededForJoins()
             if (!$logTable->getColumnToJoinOnIdVisit()) {
                 $tableNameToJoin = $logTable->getLinkTableToBeAbleToJoinOnVisit();
 
+                if (empty($tableNameToJoin) && $logTable->getWaysToJoinToOtherLogTables()) {
+                    foreach ($logTable->getWaysToJoinToOtherLogTables() as $otherLogTable => $column) {
+                        if ($this->tables->hasJoinedTable($otherLogTable)) {
+                            $this->tables->addTableDependency($table, $otherLogTable);
+                            continue;
+                        }
+                        if ($this->tables->isTableJoinableOnVisit($otherLogTable) || $this->tables->isTableJoinableOnAction($otherLogTable)) {
+                            $this->addMissingTablesForOtherTableJoin($otherLogTable, $table);
+                        }
+                    }
+                    continue;
+                }
+
                 if ($index > 0 && !$this->tables->hasJoinedTable($tableNameToJoin)) {
                     $this->tables->addTableToJoin($tableNameToJoin);
                 }
@@ -96,6 +109,30 @@ private function addMissingTablesNeededForJoins()
         }
     }
 
+    private function addMissingTablesForOtherTableJoin($tableName, $dependentTable)
+    {
+        $this->tables->addTableDependency($dependentTable, $tableName);
+
+        if ($this->tables->hasJoinedTable($tableName)) {
+            return;
+        }
+
+        $table = $this->tables->getLogTable($tableName);
+
+        if ($table->getColumnToJoinOnIdAction() || $table->getColumnToJoinOnIdAction() || $table->getLinkTableToBeAbleToJoinOnVisit()) {
+            $this->tables->addTableToJoin($tableName);
+            return;
+        }
+
+        $otherTableJoins = $table->getWaysToJoinToOtherLogTables();
+
+        foreach ($otherTableJoins as $logTable => $column) {
+            $this->addMissingTablesForOtherTableJoin($logTable, $tableName);
+        }
+
+        $this->tables->addTableToJoin($tableName);
+    }
+
     /**
      * Generate the join sql based on the needed tables
      * @throws Exception if tables can't be joined
@@ -206,6 +243,15 @@ public function findJoinCriteriasForTables(LogTable $logTable, $availableLogTabl
 
                 break;
             }
+
+            $otherJoins = $logTable->getWaysToJoinToOtherLogTables();
+            foreach ($otherJoins as $joinTable => $column) {
+                if($availableLogTable->getName() == $joinTable) {
+                    $join = sprintf("`%s`.`%s` = `%s`.`%s`", $table, $column, $availableLogTable->getName(), $column);
+                    break;
+                }
+            }
+
         }
 
         if (!isset($join)) {

core/DataAccess/LogQueryBuilder/JoinTables.php

@@ -19,6 +19,40 @@ class JoinTables extends \ArrayObject
      */
     private $logTableProvider;
 
+    // NOTE: joins can be specified explicitly as arrays w/ 'joinOn' keys or implicitly as table names. when
+    // table names are used, the joins dependencies are assumed based on how we want to order those joins.
+    // the below table list the possible dependencies of each table, and is specifically designed to enforce
+    // the following order:
+    // log_link_visit_action, log_action, log_visit, log_conversion, log_conversion_item
+    // which means if an array is supplied where log_visit comes before log_link_visitAction, it will
+    // be moved to after it.
+    private $implicitTableDependencies = [
+        'log_link_visit_action' => [
+            // empty
+        ],
+        'log_action' => [
+            'log_link_visit_action',
+            'log_conversion',
+            'log_conversion_item',
+            'log_visit',
+        ],
+        'log_visit' => [
+            'log_link_visit_action',
+            'log_action',
+        ],
+        'log_conversion' => [
+            'log_link_visit_action',
+            'log_action',
+            'log_visit',
+        ],
+        'log_conversion_item' => [
+            'log_link_visit_action',
+            'log_action',
+            'log_visit',
+            'log_conversion',
+        ],
+    ];
+
     /**
      * Tables constructor.
      * @param LogTablesProvider $logTablesProvider
@@ -125,6 +159,73 @@ public function sort()
         $this->exchangeArray($sorted);
     }
 
+    public function isTableJoinableOnVisit($tableToCheck)
+    {
+        $table = $this->getLogTable($tableToCheck);
+
+        if (empty($table)) {
+            return false;
+        }
+
+        if ($table->getColumnToJoinOnIdVisit()) {
+            return true;
+        }
+
+        if ($table->getLinkTableToBeAbleToJoinOnVisit()) {
+            return true;
+        }
+
+        $otherWays = $table->getWaysToJoinToOtherLogTables();
+
+        if (empty($otherWays)) {
+            return false;
+        }
+
+        foreach ($otherWays as $logTable => $column) {
+            if ($logTable == 'log_visit' || $this->isTableJoinableOnVisit($logTable)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    public function isTableJoinableOnAction($tableToCheck)
+    {
+        $table = $this->getLogTable($tableToCheck);
+
+        if (empty($table)) {
+            return false;
+        }
+
+        if ($table->getColumnToJoinOnIdAction()) {
+            return true;
+        }
+
+        $otherWays = $table->getWaysToJoinToOtherLogTables();
+
+        if (empty($otherWays)) {
+            return false;
+        }
+
+        foreach ($otherWays as $logTable => $column) {
+            if ($logTable == 'log_action' || $this->isTableJoinableOnAction($logTable)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    public function addTableDependency($table, $dependentTable)
+    {
+        if (!empty($this->implicitTableDependencies[$table])) {
+            return;
+        }
+
+        $this->implicitTableDependencies[$table] = [$dependentTable];
+    }
+
     private function checkTableCanBeUsedForSegmentation($tableName)
     {
         if (!is_array($tableName) && !$this->getLogTable($tableName)) {
@@ -154,39 +255,7 @@ private function parseDependencies(array $tables)
 
     private function assumeImplicitJoinDependencies($allTablesToQuery, $table)
     {
-        // NOTE: joins can be specified explicitly as arrays w/ 'joinOn' keys or implicitly as table names. when
-        // table names are used, the joins dependencies are assumed based on how we want to order those joins.
-        // the below table list the possible dependencies of each table, and is specifically designed to enforce
-        // the following order:
-        // log_link_visit_action, log_action, log_visit, log_conversion, log_conversion_item
-        // which means if an array is supplied where log_visit comes before log_link_visitAction, it will
-        // be moved to after it.
-        $implicitTableDependencies = [
-            'log_link_visit_action' => [
-                // empty
-            ],
-            'log_action' => [
-                'log_link_visit_action',
-                'log_conversion',
-                'log_conversion_item',
-                'log_visit',
-            ],
-            'log_visit' => [
-                'log_link_visit_action',
-                'log_action',
-            ],
-            'log_conversion' => [
-                'log_link_visit_action',
-                'log_action',
-                'log_visit',
-            ],
-            'log_conversion_item' => [
-                'log_link_visit_action',
-                'log_action',
-                'log_visit',
-                'log_conversion',
-            ],
-        ];
+        $implicitTableDependencies = $this->implicitTableDependencies;
 
         $result = [];
         if (isset($implicitTableDependencies[$table])) {

plugins/ExampleLogTables/Columns/GroupAttributeAdmin.php

@@ -0,0 +1,14 @@
+<?php
+namespace Piwik\Plugins\ExampleLogTables\Columns;
+
+use Piwik\Columns\Dimension;
+
+class GroupAttributeAdmin extends Dimension
+{
+    protected $dbTableName  = 'log_group';
+    protected $category     = 'General_Visitors';
+    protected $type         = self::TYPE_BOOL;
+    protected $columnName   = 'is_admin';
+    protected $segmentName  = 'isadmin';
+    protected $nameSingular = 'Admin privileges';
+}

plugins/ExampleLogTables/Columns/UserAttributeGender.php

@@ -0,0 +1,14 @@
+<?php
+namespace Piwik\Plugins\ExampleLogTables\Columns;
+
+use Piwik\Columns\Dimension;
+
+class UserAttributeGender extends Dimension
+{
+    protected $dbTableName  = 'log_custom';
+    protected $category     = 'General_Visitors';
+    protected $type         = self::TYPE_TEXT;
+    protected $columnName   = 'gender';
+    protected $segmentName  = 'attrgender';
+    protected $nameSingular = 'Gender';
+}

plugins/ExampleLogTables/Dao/CustomGroupLog.php

@@ -0,0 +1,66 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\ExampleLogTables\Dao;
+
+use Piwik\Common;
+use Piwik\Db;
+use Piwik\DbHelper;
+
+class CustomGroupLog
+{
+    private $table = 'log_group';
+    private $tablePrefixed = '';
+
+    public function __construct()
+    {
+        $this->tablePrefixed = Common::prefixTable($this->table);
+    }
+
+    public function install()
+    {
+        DbHelper::createTable($this->table, "
+                  `group` VARCHAR(30) NOT NULL,
+                  `is_admin` TINYINT(1) NOT NULL,
+                  PRIMARY KEY (`group`)");
+    }
+
+    public function uninstall()
+    {
+        Db::query(sprintf('DROP TABLE IF EXISTS `%s`', $this->tablePrefixed));
+    }
+
+    private function getDb()
+    {
+        return Db::get();
+    }
+
+    public function getAllRecords()
+    {
+        return $this->getDb()->fetchAll('SELECT * FROM ' . $this->tablePrefixed);
+    }
+
+    public function addGroupInformation($group, $isAdmin)
+    {
+        $columns = array(
+            'group' => $group,
+            'is_admin' => $isAdmin
+        );
+
+        $bind = array_values($columns);
+        $placeholder = Common::getSqlStringFieldsArray($columns);
+
+        $sql = sprintf('INSERT INTO %s (`%s`) VALUES(%s)',
+            $this->tablePrefixed, implode('`,`', array_keys($columns)), $placeholder);
+
+        $db = $this->getDb();
+
+        $db->query($sql, $bind);
+    }
+}
+

plugins/ExampleLogTables/Dao/CustomUserLog.php

@@ -0,0 +1,68 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\ExampleLogTables\Dao;
+
+use Piwik\Common;
+use Piwik\Db;
+use Piwik\DbHelper;
+
+class CustomUserLog
+{
+    private $table = 'log_custom';
+    private $tablePrefixed = '';
+
+    public function __construct()
+    {
+        $this->tablePrefixed = Common::prefixTable($this->table);
+    }
+
+    public function install()
+    {
+        DbHelper::createTable($this->table, "
+                  `user_id` VARCHAR(200) NOT NULL,
+                  `gender` VARCHAR(30) NOT NULL,
+                  `group` VARCHAR(30) NOT NULL,
+                  PRIMARY KEY (user_id)");
+    }
+
+    public function uninstall()
+    {
+        Db::query(sprintf('DROP TABLE IF EXISTS `%s`', $this->tablePrefixed));
+    }
+
+    private function getDb()
+    {
+        return Db::get();
+    }
+
+    public function getAllRecords()
+    {
+        return $this->getDb()->fetchAll('SELECT * FROM ' . $this->tablePrefixed);
+    }
+
+    public function addUserInformation($userId, $group, $gender)
+    {
+        $columns = array(
+            'user_id' => $userId,
+            'group' => $group,
+            'gender' => $gender
+        );
+
+        $bind = array_values($columns);
+        $placeholder = Common::getSqlStringFieldsArray($columns);
+
+        $sql = sprintf('INSERT INTO %s (`%s`) VALUES(%s)',
+            $this->tablePrefixed, implode('`,`', array_keys($columns)), $placeholder);
+
+        $db = $this->getDb();
+
+        $db->query($sql, $bind);
+    }
+}
+