CiviCRM 5.58.1 update

Note we have some recent patches that are included. These are mostly ones merged to
civicrm-core since forking for 5.58. One is in 5.58 now so is listed for
completeness & 2 minor ones are not yet merged upstream

|pr|status|notes|
|[25565](https://github.com/civicrm/civicrm-core/pull/25565)|merged to master(5.60)|Supports T327963 - activity types for thank you|
|[25516](https://github.com/civicrm/civicrm-core/pull/25516)|merged to master(5.60)|Helps import - filter imported searches|
|[25527](https://github.com/civicrm/civicrm-core/pull/25527)|not yet merged| but performance fix for dedupe searches (the name & address ones Sandra is doing)
|[25433](https://github.com/civicrm/civicrm-core/pull/25433)|merged to rc (5.59)|fix for search kit display|
|[25482](https://github.com/civicrm/civicrm-core/pull/25482)|Already in 5.58.1|Fix for search kit display|
|[25418](https://github.com/civicrm/civicrm-core/pull/25418)|Merged to rc (5.59)|Permanent imap timout fix|
|[25226](https://github.com/civicrm/civicrm-core/pull/25226)|Merged to rc (5.59)|ReportInstance api - in our install|
|[25568](https://github.com/civicrm/civicrm-core/pull/25568)|not yet merged|Helps with new import code, reduces chance of crash|
|[24515](https://github.com/civicrm/civicrm-core/pull/25415)|Merged to rc (5.59)| Fix to stop CI from breaking, replaces our hack|

List of cherry picked commits from the above

18d82af27b9 (HEAD -> master) Filter expired searches from search kit results
a75b98cb47a Add test to is_current on SavedSearch, fix
e4d69230ad1 Increase timeout on imap
89ab855619f Do not crash the whole SearchKit UI if one entity fails
0142e5dd18c Fix performance on deciding which tables to query
cd710af9733 dev/core#4117 Add is_current to UserJob, Search
2fa17fb88ae Make activity_type_id available to links, test
2d2bd93cda8 (dev/core#4088) ClassScanner - Move unit-test registration
e3e669d4556 Add Report Instance apiv4

Bug: T329681
Change-Id: I839d1bbfabfe3558094f7445a4281f237b609fed

drupal/sites/all/modules/civicrm/CRM/Activity/Selector/Search.php

@@ -303,6 +303,10 @@ public function &getRows($action, $offset, $rowCount, $sort, $output = NULL) {
           'id' => $result->activity_id,
           'cid' => $contactId,
           'cxt' => $this->_context,
+          // Parameter for hook locked in by CRM_Activity_Selector_SearchTest
+          // Any additional parameters added should follow apiv4 style
+          // and be added to the test.
+          'activity_type_id' => $row['activity_type_id'],
         ],
         ts('more'),
         FALSE,

drupal/sites/all/modules/civicrm/CRM/Contact/Tokens.php

@@ -397,14 +397,17 @@ protected function getTokenMetadata(): array {
       foreach ($metadata as $field) {
         if ($entity === 'website') {
           // It's not the primary - it's 'just one of them' - so the name is _first not _primary
+          $field['name'] = 'website_first.' . $field['name'];
           $this->addFieldToTokenMetadata($tokensMetadata, $field, $exposedFields, 'website_first');
         }
         else {
+          $field['name'] = $entity . '_primary.' . $field['name'];
           $this->addFieldToTokenMetadata($tokensMetadata, $field, $exposedFields, $entity . '_primary');
           $field['label'] .= ' (' . ts('Billing') . ')';
           // Set audience to sysadmin in case adding them to UI annoys people. If people ask to see this
           // in the UI we could set to 'user'.
           $field['audience'] = 'sysadmin';
+          $field['name'] = $entity . '_billing.' . $field['name'];
           $this->addFieldToTokenMetadata($tokensMetadata, $field, $exposedFields, $entity . '_billing');
         }
       }
@@ -453,13 +456,11 @@ protected function getContact(int $contactId, array $requiredFields, bool $getAl
         if ($fieldSpec['table_name'] === 'civicrm_website') {
           $tableAlias = 'website_first';
           $joins[$tableAlias] = $fieldSpec['entity'];
-          $prefix = $tableAlias . '.';
         }
         if ($fieldSpec['table_name'] === 'civicrm_openid') {
           // We could start to deprecate this one maybe..... I've made it un-advertised.
           $tableAlias = 'openid_primary';
           $joins[$tableAlias] = $fieldSpec['entity'];
-          $prefix = $tableAlias . '.';
         }
         if ($fieldSpec['type'] === 'Custom') {
           $customFields['custom_' . $fieldSpec['custom_field_id']] = $fieldSpec['name'];

drupal/sites/all/modules/civicrm/CRM/Contribute/Form/Task/Invoice.php

@@ -233,10 +233,9 @@ public static function printPDF($contribIDs, &$params, $contactIds) {
     $refundedStatusId = CRM_Utils_Array::key('Refunded', $contributionStatusID);
     $cancelledStatusId = CRM_Utils_Array::key('Cancelled', $contributionStatusID);
     $pendingStatusId = CRM_Utils_Array::key('Pending', $contributionStatusID);
-    $pdfFormat = CRM_Core_BAO_PdfFormat::getByName('default_invoice_pdf_format');
-
+    $pdfFormat = CRM_Core_BAO_MessageTemplate::getPDFFormatForTemplate('contribution_invoice_receipt');
     foreach ($elementDetails as $contributionID => $detail) {
-      $input = $ids = [];
+      $input = [];
       if (in_array($detail['contact'], $excludedContactIDs)) {
         continue;
       }

drupal/sites/all/modules/civicrm/CRM/Core/BAO/MessageTemplate.php

@@ -203,7 +203,7 @@ public static function self_hook_civicrm_pre(\Civi\Core\Event\PreEvent $event) {
    */
   public static function getMessageTemplates($all = TRUE, $isSMS = FALSE) {
 
-    $messageTemplates = \Civi\Api4\MessageTemplate::get()
+    $messageTemplates = MessageTemplate::get()
       ->addSelect('id', 'msg_title')
       ->addWhere('is_active', '=', TRUE)
       ->addWhere('is_sms', '=', $isSMS);
@@ -218,6 +218,24 @@ public static function getMessageTemplates($all = TRUE, $isSMS = FALSE) {
     return $msgTpls;
   }
 
+  /**
+   * Get the appropriate pdf format for the given template.
+   *
+   * @param string $workflow
+   *
+   * @return array
+   * @throws \CRM_Core_Exception
+   */
+  public static function getPDFFormatForTemplate(string $workflow): array {
+    $pdfFormatID = MessageTemplate::get(FALSE)
+      ->addWhere('workflow_name', '=', $workflow)
+      ->addSelect('pdf_format_id')
+      ->execute()->first()['pdf_format_id'] ?? 0;
+    // Get by ID will fall back to retrieving the default values if
+    // it does not find the appropriate ones - hence passing in 0 works.
+    return CRM_Core_BAO_PdfFormat::getById($pdfFormatID);
+  }
+
   /**
    * Revert a message template to its default subject+text+HTML state.
    *

drupal/sites/all/modules/civicrm/CRM/Core/BAO/PdfFormat.php

@@ -230,6 +230,7 @@ public static function &getPdfFormat($field, $val) {
    *   (reference) associative array of name/value pairs
    */
   public static function &getByName($name) {
+    CRM_Core_Error::deprecatedFunctionWarning('none');
     return self::getPdfFormat('name', $name);
   }
 

drupal/sites/all/modules/civicrm/CRM/Core/BAO/SchemaHandler.php

@@ -906,6 +906,49 @@ public static function getInUseCollation(): string {
     return \Civi::$statics[__CLASS__][__FUNCTION__];
   }
 
+  /**
+   * Get estimated number of rows in the given tables.
+   *
+   * Note that this query is less precise than SELECT(*) - especially on
+   * larger tables but performs significantly better.
+   * See https://dba.stackexchange.com/questions/184685/why-is-count-slow-when-explain-knows-the-answer
+   *
+   * @param array $tables
+   *   e.g ['civicrm_contact', 'civicrm_activity']
+   *
+   * @return array
+   *   e.g ['civicrm_contact' => 200000, 'civicrm_activity' => 100000]
+   */
+  public static function getRowCountForTables(array $tables): array {
+    $cachedResults = Civi::$statics[__CLASS__][__FUNCTION__] ?? [];
+    // Compile list of tables not already cached.
+    $tablesToCheck = array_keys(array_diff_key(array_flip($tables), $cachedResults));
+    $result = CRM_Core_DAO::executeQuery('
+      SELECT TABLE_ROWS as row_count, TABLE_NAME as table_name FROM information_schema.TABLES WHERE
+      TABLE_NAME IN("' . implode('","', $tablesToCheck) . '")
+      AND TABLE_SCHEMA = DATABASE()'
+    );
+    while ($result->fetch()) {
+      $cachedResults[$result->table_name] = (int) $result->row_count;
+    }
+    Civi::$statics[__CLASS__][__FUNCTION__] = $cachedResults;
+    return array_intersect_key($cachedResults, array_fill_keys($tables, TRUE));
+  }
+
+  /**
+   * Get estimated number of rows in the given table.
+   *
+   * @see self::getRowCountForTables
+   *
+   * @param string $tableName
+   *
+   * @return int
+   *   The approximate number of rows in the table. This is also 0 if the table does not exist.
+   */
+  public static function getRowCountForTable(string $tableName): int {
+    return self::getRowCountForTables([$tableName])[$tableName] ?? 0;
+  }
+
   /**
    * Does the database support utf8mb4.
    *

drupal/sites/all/modules/civicrm/CRM/Core/DAO.php

@@ -1100,7 +1100,7 @@ public static function checkFieldExists($tableName, $columnName, $i18nRewrite =
   }
 
   /**
-   * Scans all the tables using a slow query and table name.
+   * Gets the names of all the tables in the schema.
    *
    * @return array
    */

drupal/sites/all/modules/civicrm/CRM/Core/EntityTokens.php

@@ -611,7 +611,8 @@ public function getActiveTokens(TokenValueEvent $e) {
    * @param string $prefix
    */
   protected function addFieldToTokenMetadata(array &$tokensMetadata, array $field, array $exposedFields, string $prefix = ''): void {
-    if ($field['type'] !== 'Custom' && !in_array($field['name'], $exposedFields, TRUE)) {
+    $isExposed = in_array(str_replace($prefix . '.', '', $field['name']), $exposedFields, TRUE);
+    if ($field['type'] !== 'Custom' && !$isExposed) {
       return;
     }
     $field['audience'] = $field['audience'] ?? 'user';
@@ -635,8 +636,9 @@ protected function addFieldToTokenMetadata(array &$tokensMetadata, array $field,
       $tokensMetadata[$tokenName] = $field;
       return;
     }
-    $tokenName = $prefix ? ($prefix . '.' . $field['name']) : $field['name'];
-    if (in_array($field['name'], $exposedFields, TRUE)) {
+    $tokenName = $field['name'];
+    // Presumably this line can not be reached unless isExposed = TRUE.
+    if ($isExposed) {
       if (
         ($field['options'] || !empty($field['suffixes']))
         // At the time of writing currency didn't have a label option - this may have changed.

drupal/sites/all/modules/civicrm/CRM/Dedupe/BAO/DedupeRuleGroup.php

@@ -263,7 +263,7 @@ public function fillTable() {
           $query = "{$insertClause} {$query} {$groupByClause} ON DUPLICATE KEY UPDATE weight = weight + VALUES(weight)";
           $dao = CRM_Core_DAO::executeQuery($query);
 
-          // FIXME: we need to be more acurate with affected rows, especially for insert vs duplicate insert.
+          // FIXME: we need to be more accurate with affected rows, especially for insert vs duplicate insert.
           // And that will help optimize further.
           $affectedRows = $dao->affectedRows();
 
@@ -337,28 +337,31 @@ public static function isQuerySetInclusive($tableQueries, $threshold, $exclWeigh
   }
 
   /**
-   * sort queries by number of records for the table associated with them.
-   * @param $tableQueries
+   * Sort queries by number of records for the table associated with them.
+   *
+   * @param array $tableQueries
    */
-  public static function orderByTableCount(&$tableQueries) {
-    static $tableCount = [];
-
-    $tempArray = [];
-    foreach ($tableQueries as $key => $query) {
-      $table = explode(".", $key);
-      $table = $table[0];
-      if (!array_key_exists($table, $tableCount)) {
-        $query = "SELECT COUNT(*) FROM {$table}";
-        $tableCount[$table] = CRM_Core_DAO::singleValueQuery($query);
-      }
-      $tempArray[$key] = $tableCount[$table];
-    }
+  public static function orderByTableCount(array &$tableQueries): void {
+    uksort($tableQueries, 'self::isTableBigger');
+  }
 
-    asort($tempArray);
-    foreach ($tempArray as $key => $count) {
-      $tempArray[$key] = $tableQueries[$key];
+  /**
+   * Is the table extracted from the first string larger than the second string.
+   *
+   * @param string $a
+   *   e.g civicrm_contact.first_name
+   * @param string $b
+   *   e.g civicrm_address.street_address
+   *
+   * @return int
+   */
+  private static function isTableBigger(string $a, string $b): int {
+    $tableA = explode('.', $a)[0];
+    $tableB = explode('.', $b)[0];
+    if ($tableA === $tableB) {
+      return 0;
     }
-    $tableQueries = $tempArray;
+    return CRM_Core_BAO_SchemaHandler::getRowCountForTable($tableA) <=> CRM_Core_BAO_SchemaHandler::getRowCountForTable($tableB);
   }
 
   /**

drupal/sites/all/modules/civicrm/CRM/Event/Form/ManageEvent/Registration.php

@@ -444,7 +444,8 @@ public static function formRule($values, $files, $form) {
       if (($values['registration_link_text'] ?? '') === '') {
         $errorMsg['registration_link_text'] = ts('Please enter Registration Link Text');
       }
-      if (($values['confirm_title'] ?? '') === '') {
+      // Check if the confirm text is set if we have enabled the confirmation page or page is monetary which forces the confirm page.
+      if (($values['confirm_title'] ?? '') === '' && (!empty($values['is_confirm_enabled']) || CRM_Core_DAO::getFieldValue('CRM_Event_DAO_Event', $form->_id, 'is_monetary'))) {
         $errorMsg['confirm_title'] = ts('Please enter a Title for the registration Confirmation Page');
       }
       if (($values['thankyou_title'] ?? '') === '') {

drupal/sites/all/modules/civicrm/CRM/Extension/ClassLoader.php

@@ -179,7 +179,12 @@ private static function loadExtension(\Composer\Autoload\ClassLoader $loader, CR
    * @return string
    */
   protected function getCacheFile() {
-    $envId = \CRM_Core_Config_Runtime::getId();
+    $envId = md5(implode(',', array_merge(
+      [\CRM_Core_Config_Runtime::getId()],
+      array_column($this->mapper->getActiveModuleFiles(), 'prefix')
+      // dev/core#4055 - When toggling ext's on systems with opcode caching, you may get stale reads for a moment.
+      // New cache key ensures new data-set.
+    )));
     $file = \Civi::paths()->getPath("[civicrm.compile]/CachedExtLoader.{$envId}.php");
     return $file;
   }

drupal/sites/all/modules/civicrm/CRM/Report/Form.php

@@ -3770,14 +3770,18 @@ public function legacySlowGroupFilterClause($field, $value, $op) {
     }
 
     CRM_Contact_BAO_GroupContactCache::check($smartGroups);
-
+    $aclFilter = NULL;
+    $selectWhereClauses = array_filter(CRM_Contact_BAO_Group::getSelectWhereClause('group'));
+    $aclFilter = implode(' AND ', $selectWhereClauses);
+    $aclFilter = !empty($aclFilter) ? ' AND ' . $aclFilter : '';
     $smartGroupQuery = '';
     if (!empty($smartGroups)) {
       $smartGroups = implode(',', $smartGroups);
       $smartGroupQuery = " UNION DISTINCT
                   SELECT DISTINCT smartgroup_contact.contact_id
                   FROM civicrm_group_contact_cache smartgroup_contact
-                  WHERE smartgroup_contact.group_id IN ({$smartGroups}) ";
+                  INNER JOIN `civicrm_group` AS `group` ON `group`.id = smartgroup_contact.group_id
+                  WHERE smartgroup_contact.group_id IN ({$smartGroups}) {$aclFilter}";
     }
 
     $sqlOp = $this->getSQLOperator($op);
@@ -3796,7 +3800,8 @@ public function legacySlowGroupFilterClause($field, $value, $op) {
     return " {$contactAlias}.id {$sqlOp} (
                           SELECT DISTINCT {$this->_aliases['civicrm_group']}.contact_id
                           FROM civicrm_group_contact {$this->_aliases['civicrm_group']}
-                          WHERE {$clause} AND {$this->_aliases['civicrm_group']}.status = 'Added'
+                          INNER JOIN `civicrm_group` AS `group` ON `group`.id = {$this->_aliases['civicrm_group']}.group_id
+                          WHERE {$clause} AND {$this->_aliases['civicrm_group']}.status = 'Added' {$aclFilter}
                           {$smartGroupQuery} ) ";
   }
 
@@ -3950,6 +3955,10 @@ public function buildPermissionClause() {
     $ret = [];
     foreach ($this->selectedTables() as $tableName) {
       $baoName = str_replace('_DAO_', '_BAO_', (CRM_Core_DAO_AllCoreTables::getClassForTable($tableName) ?? ''));
+      // Do not include CiviCRM group add Select Where clause because we don't necessarily join here for reports with optimisedGroupFilters
+      if ($baoName === 'CRM_Contact_BAO_Group') {
+        continue;
+      }
       if ($baoName && class_exists($baoName) && !empty($this->_columns[$tableName]['alias'])) {
         $tableAlias = $this->_columns[$tableName]['alias'];
         $clauses = array_filter($baoName::getSelectWhereClause($tableAlias));

drupal/sites/all/modules/civicrm/CRM/Utils/Check/Component/Timestamps.php

@@ -58,7 +58,7 @@ public function checkTimezoneAPIs() {
           1 => 'target="_blank" href="https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html"',
         ]),
         ts('MySQL Timezone Problem'),
-        \Psr\Log\LogLevel::WARNING,
+        \Psr\Log\LogLevel::NOTICE,
         'fa-clock-o'
       );
     }

drupal/sites/all/modules/civicrm/CRM/Utils/File.php

@@ -352,8 +352,7 @@ public static function stripComments($string) {
    * @return bool
    */
   public static function isExtensionSafe($ext) {
-    static $extensions = NULL;
-    if (!$extensions) {
+    if (!isset(Civi::$statics[__CLASS__]['file_extensions'])) {
       $extensions = CRM_Core_OptionGroup::values('safe_file_extension', TRUE);
 
       // make extensions to lowercase
@@ -370,9 +369,11 @@ public static function isExtensionSafe($ext) {
         unset($extensions['html']);
         unset($extensions['htm']);
       }
+      Civi::$statics[__CLASS__]['file_extensions'] = $extensions;
     }
+    $restricted = CRM_Utils_Constant::value('CIVICRM_RESTRICTED_UPLOADS', '/(php|php\d|phtml|phar|pl|py|cgi|asp|js|sh|exe|pcgi\d)/i');
     // support lower and uppercase file extensions
-    return (bool) isset($extensions[strtolower($ext)]);
+    return (bool) isset(Civi::$statics[__CLASS__]['file_extensions'][strtolower($ext)]) && !preg_match($restricted, strtolower($ext));
   }
 
   /**

drupal/sites/all/modules/civicrm/CRM/Utils/Hook.php

@@ -309,6 +309,10 @@ public function runHooks(
    * @param $moduleList
    */
   public function requireCiviModules(&$moduleList) {
+    foreach ($GLOBALS['CIVICRM_FORCE_MODULES'] ?? [] as $prefix) {
+      $moduleList[$prefix] = $prefix;
+    }
+
     $civiModules = CRM_Core_PseudoConstant::getModuleExtensions();
     foreach ($civiModules as $civiModule) {
       if (!file_exists($civiModule['filePath'] ?? '')) {