From fbdb1ee2b15db21d348286b0c0ffa52150d7ad30 Mon Sep 17 00:00:00 2001 From: colemanw Date: Thu, 15 Jun 2023 17:22:45 -0400 Subject: [PATCH] APIv4 - Add EntitySet api for set-ops such as UNION --- Civi/Api4/Action/EntitySet/Get.php | 62 +++ Civi/Api4/EntitySet.php | 53 ++ Civi/Api4/Generic/BasicGetAction.php | 4 +- Civi/Api4/Generic/DAOGetAction.php | 59 +-- .../Generic/Traits/CustomValueActionTrait.php | 4 +- Civi/Api4/Generic/Traits/DAOActionTrait.php | 4 +- .../Traits/GroupAndHavingParamTrait.php | 66 +++ Civi/Api4/Query/Api4EntitySetQuery.php | 179 +++++++ Civi/Api4/Query/Api4Query.php | 496 ++++++++++++++++++ Civi/Api4/Query/Api4SelectQuery.php | 477 +---------------- Civi/Api4/Query/SqlEquation.php | 4 +- Civi/Api4/Query/SqlExpression.php | 4 +- Civi/Api4/Query/SqlField.php | 4 +- Civi/Api4/Query/SqlFunction.php | 8 +- Civi/Api4/Query/SqlNull.php | 2 +- Civi/Api4/Query/SqlNumber.php | 2 +- Civi/Api4/Query/SqlString.php | 2 +- Civi/Api4/Query/SqlWild.php | 2 +- Civi/Api4/Utils/FormattingUtil.php | 82 ++- .../Api4/Action/SearchDisplay/GetDefault.php | 5 +- .../api/v4/Action/EntitySetUnionTest.php | 120 +++++ 21 files changed, 1050 insertions(+), 589 deletions(-) create mode 100644 Civi/Api4/Action/EntitySet/Get.php create mode 100644 Civi/Api4/EntitySet.php create mode 100644 Civi/Api4/Generic/Traits/GroupAndHavingParamTrait.php create mode 100644 Civi/Api4/Query/Api4EntitySetQuery.php create mode 100644 Civi/Api4/Query/Api4Query.php create mode 100644 tests/phpunit/api/v4/Action/EntitySetUnionTest.php diff --git a/Civi/Api4/Action/EntitySet/Get.php b/Civi/Api4/Action/EntitySet/Get.php new file mode 100644 index 000000000000..d39e9250f694 --- /dev/null +++ b/Civi/Api4/Action/EntitySet/Get.php @@ -0,0 +1,62 @@ +sets[] = [$type, $apiRequest->getEntityName(), $apiRequest->getActionName(), $apiRequest->getParams()]; + return $this; + } + + /** + * @throws \CRM_Core_Exception + */ + public function _run(Result $result) { + $query = new Api4EntitySetQuery($this); + $rows = $query->run(); + \CRM_Utils_API_HTMLInputCoder::singleton()->decodeRows($rows); + $result->exchangeArray($rows); + } + +} diff --git a/Civi/Api4/EntitySet.php b/Civi/Api4/EntitySet.php new file mode 100644 index 000000000000..03ac75665020 --- /dev/null +++ b/Civi/Api4/EntitySet.php @@ -0,0 +1,53 @@ +setCheckPermissions($checkPermissions); + } + + /** + * @return \Civi\Api4\Generic\BasicGetFieldsAction + */ + public static function getFields($checkPermissions = TRUE) { + return (new BasicGetFieldsAction('EntitySet', __FUNCTION__, function() { + return []; + }))->setCheckPermissions($checkPermissions); + } + + public static function permissions() { + return []; + } + + /** + * @param bool $plural + * @return string + */ + protected static function getEntityTitle($plural = FALSE) { + return $plural ? ts('Entity Sets') : ts('Entity Set'); + } + +} diff --git a/Civi/Api4/Generic/BasicGetAction.php b/Civi/Api4/Generic/BasicGetAction.php index 95639ae36f0e..15c9951d8a29 100644 --- a/Civi/Api4/Generic/BasicGetAction.php +++ b/Civi/Api4/Generic/BasicGetAction.php @@ -116,9 +116,9 @@ protected function formatRawValues(&$records) { } } } + // Swap raw values with pseudoconstants + FormattingUtil::formatOutputValues($values, $fields, $this->getActionName()); } - // Swap raw values with pseudoconstants - FormattingUtil::formatOutputValues($records, $fields, $this->getActionName()); } } diff --git a/Civi/Api4/Generic/DAOGetAction.php b/Civi/Api4/Generic/DAOGetAction.php index 0e85f84d7b35..b2620759a37a 100644 --- a/Civi/Api4/Generic/DAOGetAction.php +++ b/Civi/Api4/Generic/DAOGetAction.php @@ -22,13 +22,12 @@ * * Perform joins on other related entities using a dot notation. * - * @method $this setHaving(array $clauses) - * @method array getHaving() * @method $this setTranslationMode(string|null $mode) * @method string|null getTranslationMode() */ class DAOGetAction extends AbstractGetAction { use Traits\DAOActionTrait; + use Traits\GroupAndHavingParamTrait; /** * Fields to return. Defaults to all standard (non-custom, non-extra) fields `['*']`. @@ -66,22 +65,6 @@ class DAOGetAction extends AbstractGetAction { */ protected $join = []; - /** - * Field(s) by which to group the results. - * - * @var array - */ - protected $groupBy = []; - - /** - * Clause for filtering results after grouping and filters are applied. - * - * Each expression should correspond to an item from the SELECT array. - * - * @var array - */ - protected $having = []; - /** * Should we automatically overload the result with translated data? * How do we pick the suitable translation? @@ -160,46 +143,6 @@ public function addWhere(string $fieldName, string $op, $value = NULL, bool $isE return $this; } - /** - * @return array - */ - public function getGroupBy(): array { - return $this->groupBy; - } - - /** - * @param array $groupBy - * @return $this - */ - public function setGroupBy(array $groupBy) { - $this->groupBy = $groupBy; - return $this; - } - - /** - * @param string $field - * @return $this - */ - public function addGroupBy(string $field) { - $this->groupBy[] = $field; - return $this; - } - - /** - * @param string $expr - * @param string $op - * @param mixed $value - * @return $this - * @throws \CRM_Core_Exception - */ - public function addHaving(string $expr, string $op, $value = NULL) { - if (!in_array($op, CoreUtil::getOperators())) { - throw new \CRM_Core_Exception('Unsupported operator'); - } - $this->having[] = [$expr, $op, $value]; - return $this; - } - /** * @param string $entity * @param string|bool $type diff --git a/Civi/Api4/Generic/Traits/CustomValueActionTrait.php b/Civi/Api4/Generic/Traits/CustomValueActionTrait.php index e194ef6af4c9..14159ad18cd0 100644 --- a/Civi/Api4/Generic/Traits/CustomValueActionTrait.php +++ b/Civi/Api4/Generic/Traits/CustomValueActionTrait.php @@ -65,6 +65,8 @@ public function isAuthorized(): bool { */ protected function writeObjects($items) { $fields = $this->entityFields(); + // Note: Some parts of this loop mutate $item for purposes of internal processing only + // so we do not loop through $items by reference as to preserve the original structure for output. foreach ($items as $idx => $item) { FormattingUtil::formatWriteParams($item, $fields); @@ -83,8 +85,8 @@ protected function writeObjects($items) { $tableName = CoreUtil::getTableName($this->getEntityName()); $items[$idx]['id'] = (int) \CRM_Core_DAO::singleValueQuery('SELECT MAX(id) FROM ' . $tableName); } + FormattingUtil::formatOutputValues($items[$idx], $fields, 'create'); } - FormattingUtil::formatOutputValues($items, $this->entityFields(), 'create'); return $items; } diff --git a/Civi/Api4/Generic/Traits/DAOActionTrait.php b/Civi/Api4/Generic/Traits/DAOActionTrait.php index 55785c8af0da..3de18daa7ca6 100644 --- a/Civi/Api4/Generic/Traits/DAOActionTrait.php +++ b/Civi/Api4/Generic/Traits/DAOActionTrait.php @@ -140,7 +140,9 @@ protected function writeObjects($items) { } \CRM_Utils_API_HTMLInputCoder::singleton()->decodeRows($result); - FormattingUtil::formatOutputValues($result, $this->entityFields()); + foreach ($result as &$row) { + FormattingUtil::formatOutputValues($row, $this->entityFields()); + } return $result; } diff --git a/Civi/Api4/Generic/Traits/GroupAndHavingParamTrait.php b/Civi/Api4/Generic/Traits/GroupAndHavingParamTrait.php new file mode 100644 index 000000000000..73233c13552d --- /dev/null +++ b/Civi/Api4/Generic/Traits/GroupAndHavingParamTrait.php @@ -0,0 +1,66 @@ +groupBy[] = $field; + return $this; + } + + /** + * @param string $expr + * @param string $op + * @param mixed $value + * @return $this + * @throws \CRM_Core_Exception + */ + public function addHaving(string $expr, string $op, $value = NULL) { + if (!in_array($op, CoreUtil::getOperators())) { + throw new \CRM_Core_Exception('Unsupported operator'); + } + $this->having[] = [$expr, $op, $value]; + return $this; + } + +} diff --git a/Civi/Api4/Query/Api4EntitySetQuery.php b/Civi/Api4/Query/Api4EntitySetQuery.php new file mode 100644 index 000000000000..4c0b32e66eb5 --- /dev/null +++ b/Civi/Api4/Query/Api4EntitySetQuery.php @@ -0,0 +1,179 @@ +query = \CRM_Utils_SQL_Select::fromSet(); + $isAggregate = $this->isAggregateQuery(); + + foreach ($api->getSets() as $index => $set) { + [$type, $entity, $action, $params] = $set + [NULL, NULL, 'get', []]; + $params['checkPermissions'] = $api->getCheckPermissions(); + $params['version'] = 4; + $apiRequest = Request::create($entity, $action, $params); + // For non-aggregated queries, add a tracking id so the rows can be identified + // for output-formatting purposes + if (!$isAggregate) { + $apiRequest->addSelect($index . ' AS _api_set_index'); + } + $selectQuery = new Api4SelectQuery($apiRequest); + $selectQuery->forceSelectId = FALSE; + $selectQuery->getSql(); + // Update field aliases of all subqueries to match the first query + if ($index) { + $selectQuery->selectAliases = array_combine(array_keys($this->getSubquery()->selectAliases), $selectQuery->selectAliases); + } + $this->subqueries[] = [$type, $selectQuery]; + } + } + + /** + * Why walk when you can + * + * @return array + */ + public function run(): array { + $results = $this->getResults(); + foreach ($results as &$result) { + // Format fields based on which set this row belongs to + // This index is only available for non-aggregated queries + $index = $result['_api_set_index'] ?? NULL; + unset($result['_api_set_index']); + if (isset($index)) { + $fieldSpec = $this->getSubquery($index)->apiFieldSpec; + $selectAliases = $this->getSubquery($index)->selectAliases; + } + // Aggregated queries will have to make due with limited field info + else { + $fieldSpec = $this->apiFieldSpec; + $selectAliases = $this->selectAliases; + } + FormattingUtil::formatOutputValues($result, $fieldSpec, 'get', $selectAliases); + } + return $results; + } + + private function getSubquery(int $index = 0): Api4SelectQuery { + return $this->subqueries[$index][1]; + } + + /** + * Select * from all sets + */ + protected function buildSelectClause() { + // Default is to SELECT * FROM (subqueries) + $select = $this->api->getSelect(); + if ($select === ['*']) { + $select = []; + } + // Add all subqueries to the FROM clause + foreach ($this->subqueries as $index => $set) { + [$type, $selectQuery] = $set; + + $this->query->setOp($type, [$selectQuery->getQuery()]); + // If this outer query uses the default of SELECT * then effectively we are selecting + // all the fields of the first subquery + if (!$index && !$select) { + $this->selectAliases = $selectQuery->selectAliases; + $this->apiFieldSpec = $selectQuery->apiFieldSpec; + } + } + // Parse select clause if not using default of * + foreach ($select as $item) { + $expr = SqlExpression::convert($item, TRUE); + foreach ($expr->getFields() as $fieldName) { + $field = $this->getField($fieldName); + $this->apiFieldSpec[$fieldName] = $field; + } + $alias = $expr->getAlias(); + $this->selectAliases[$alias] = $expr->getExpr(); + $this->query->select($expr->render($this) . " AS `$alias`"); + } + } + + public function getField($expr, $strict = FALSE) { + $col = strpos($expr, ':'); + $fieldName = $col ? substr($expr, 0, $col) : $expr; + return $this->apiFieldSpec[$fieldName] ?? $this->getSubquery()->getField($expr, $strict); + } + + protected function buildWhereClause() { + foreach ($this->getWhere() as $clause) { + $sql = $this->treeWalkClauses($clause, 'HAVING'); + if ($sql) { + $this->query->where($sql); + } + } + } + + /** + * Add HAVING clause to query + * + * Every expression referenced must also be in the SELECT clause. + */ + protected function buildHavingClause() { + foreach ($this->getHaving() as $clause) { + $sql = $this->treeWalkClauses($clause, 'HAVING'); + if ($sql) { + $this->query->having($sql); + } + } + } + + /** + * Add ORDER BY to query + */ + protected function buildOrderBy() { + foreach ($this->getOrderBy() as $item => $dir) { + if ($dir !== 'ASC' && $dir !== 'DESC') { + throw new \CRM_Core_Exception("Invalid sort direction. Cannot order by $item $dir"); + } + $expr = $this->getExpression($item); + $column = $this->renderExpr($expr); + $this->query->orderBy("$column $dir"); + } + } + + /** + * Returns rendered expression or alias if it is already aliased in the SELECT clause. + * + * @param $expr + * @return mixed|string + */ + protected function renderExpr($expr) { + $exprVal = explode(':', $expr->getExpr())[0]; + // If this expression is already aliased in the select clause, use the existing alias. + foreach ($this->selectAliases as $alias => $selectVal) { + $selectVal = explode(':', $selectVal)[0]; + if ($exprVal === $selectVal) { + return "`$alias`"; + } + } + return $expr->render($this); + } + +} diff --git a/Civi/Api4/Query/Api4Query.php b/Civi/Api4/Query/Api4Query.php new file mode 100644 index 000000000000..ec71a7f87ec9 --- /dev/null +++ b/Civi/Api4/Query/Api4Query.php @@ -0,0 +1,496 @@ +=', '>', '<', 'LIKE', "<>", "!=", + * * 'NOT LIKE', 'IN', 'NOT IN', 'BETWEEN', 'NOT BETWEEN', + * * 'IS NOT NULL', 'IS NULL', 'CONTAINS', 'NOT CONTAINS', + * * 'IS EMPTY', 'IS NOT EMPTY', 'REGEXP', 'NOT REGEXP'. + */ +abstract class Api4Query { + + const + MAIN_TABLE_ALIAS = 'a', + UNLIMITED = '18446744073709551615'; + + /** + * @var \CRM_Utils_SQL_Select + */ + protected $query; + + /** + * @var \Civi\Api4\Generic\AbstractQueryAction + */ + protected $api; + + /** + * @var array + * [alias => expr][] + */ + public $selectAliases = []; + + /** + * @var array + */ + protected $entityValues = []; + + /** + * @var array[] + */ + public $apiFieldSpec = []; + + /** + * @param \Civi\Api4\Generic\AbstractQueryAction $api + */ + public function __construct($api) { + $this->api = $api; + } + + /** + * Builds main final sql statement after initialization. + * + * @return string + * @throws \CRM_Core_Exception + */ + public function getSql() { + $this->buildSelectClause(); + $this->buildWhereClause(); + $this->buildOrderBy(); + $this->buildLimit(); + $this->buildGroupBy(); + $this->buildHavingClause(); + return $this->query->toSQL(); + } + + public function getResults(): array { + $results = []; + $sql = $this->getSql(); + $this->debug('sql', $sql); + $query = \CRM_Core_DAO::executeQuery($sql); + while ($query->fetch()) { + $result = []; + foreach ($this->selectAliases as $alias => $expr) { + $returnName = $alias; + $alias = str_replace('.', '_', $alias); + $result[$returnName] = property_exists($query, $alias) ? $query->$alias : NULL; + } + $results[] = $result; + } + return $results; + } + + protected function isAggregateQuery() { + if ($this->getGroupBy()) { + return TRUE; + } + foreach ($this->getSelect() as $sql) { + $classname = get_class(SqlExpression::convert($sql, TRUE)); + if (method_exists($classname, 'getCategory') && $classname::getCategory() === SqlFunction::CATEGORY_AGGREGATE) { + return TRUE; + } + } + return FALSE; + } + + /** + * Add LIMIT to query + * + * @throws \CRM_Core_Exception + */ + protected function buildLimit() { + if ($this->getLimit() || $this->getOffset()) { + // If limit is 0, mysql will actually return 0 results. Instead set to maximum possible. + $this->query->limit($this->getLimit() ?: self::UNLIMITED, $this->getOffset()); + } + } + + /** + * Add GROUP BY clause to query + */ + protected function buildGroupBy() { + foreach ($this->getGroupBy() as $item) { + $this->query->groupBy($this->renderExpr($this->getExpression($item))); + } + } + + /** + * @param string $path + * @param array $field + */ + public function addSpecField($path, $field) { + // Only add field to spec if we have permission + if ($this->getCheckPermissions() && !empty($field['permission']) && !\CRM_Core_Permission::check($field['permission'])) { + $this->apiFieldSpec[$path] = FALSE; + return; + } + $this->apiFieldSpec[$path] = $field + [ + 'implicit_join' => NULL, + 'explicit_join' => NULL, + ]; + } + + /** + * @param string $expr + * @param array $allowedTypes + * @return SqlExpression + * @throws \CRM_Core_Exception + */ + protected function getExpression(string $expr, $allowedTypes = NULL) { + $sqlExpr = SqlExpression::convert($expr, FALSE, $allowedTypes); + foreach ($sqlExpr->getFields() as $fieldName) { + $this->getField($fieldName, TRUE); + } + return $sqlExpr; + } + + /** + * Recursively validate and transform a branch or leaf clause array to SQL. + * + * @param array $clause + * @param string $type + * WHERE|HAVING|ON + * @param int $depth + * @return string SQL where clause + * + * @throws \CRM_Core_Exception + * @uses composeClause() to generate the SQL etc. + */ + public function treeWalkClauses($clause, $type, $depth = 0) { + // Skip empty leaf. + if (in_array($clause[0], ['AND', 'OR', 'NOT']) && empty($clause[1])) { + return ''; + } + switch ($clause[0]) { + case 'OR': + case 'AND': + // handle branches + if (count($clause[1]) === 1) { + // a single set so AND|OR is immaterial + return $this->treeWalkClauses($clause[1][0], $type, $depth + 1); + } + else { + $sql_subclauses = []; + foreach ($clause[1] as $subclause) { + $sql_subclauses[] = $this->treeWalkClauses($subclause, $type, $depth + 1); + } + return '(' . implode("\n" . $clause[0] . ' ', $sql_subclauses) . ')'; + } + + case 'NOT': + // If we get a group of clauses with no operator, assume AND + if (!is_string($clause[1][0])) { + $clause[1] = ['AND', $clause[1]]; + } + return 'NOT (' . $this->treeWalkClauses($clause[1], $type, $depth + 1) . ')'; + + default: + try { + return $this->composeClause($clause, $type, $depth); + } + // Silently ignore fields the user lacks permission to see + catch (UnauthorizedException $e) { + return ''; + } + } + } + + /** + * Validate and transform a leaf clause array to SQL. + * @param array $clause [$fieldName, $operator, $criteria, $isExpression] + * @param string $type + * WHERE|HAVING|ON + * @param int $depth + * @return string SQL + * @throws \CRM_Core_Exception + * @throws \Exception + */ + public function composeClause(array $clause, string $type, int $depth) { + $field = NULL; + // Pad array for unary operators + [$expr, $operator, $value] = array_pad($clause, 3, NULL); + $isExpression = $clause[3] ?? FALSE; + if (!in_array($operator, CoreUtil::getOperators(), TRUE)) { + throw new \CRM_Core_Exception('Illegal operator'); + } + + // For WHERE clause, expr must be the name of a field. + if ($type === 'WHERE' && !$isExpression) { + $expr = $this->getExpression($expr, ['SqlField', 'SqlFunction', 'SqlEquation']); + if ($expr->getType() === 'SqlField') { + $fieldName = count($expr->getFields()) === 1 ? $expr->getFields()[0] : NULL; + $field = $this->getField($fieldName, TRUE); + FormattingUtil::formatInputValue($value, $fieldName, $field, $this->entityValues, $operator); + } + elseif ($expr->getType() === 'SqlFunction') { + $fauxField = [ + 'name' => NULL, + 'data_type' => $expr::getDataType(), + ]; + FormattingUtil::formatInputValue($value, NULL, $fauxField, $this->entityValues, $operator); + } + $fieldAlias = $expr->render($this); + } + // For HAVING, expr must be an item in the SELECT clause + elseif ($type === 'HAVING') { + // Expr references a fieldName or alias + if (isset($this->selectAliases[$expr])) { + $fieldAlias = $expr; + // Attempt to format if this is a real field + if (isset($this->apiFieldSpec[$expr])) { + $field = $this->getField($expr); + FormattingUtil::formatInputValue($value, $expr, $field, $this->entityValues, $operator); + } + } + // Expr references a non-field expression like a function; convert to alias + elseif (in_array($expr, $this->selectAliases)) { + $fieldAlias = array_search($expr, $this->selectAliases); + } + // If either the having or select field contains a pseudoconstant suffix, match and perform substitution + else { + [$fieldName] = explode(':', $expr); + foreach ($this->selectAliases as $selectAlias => $selectExpr) { + [$selectField] = explode(':', $selectAlias); + if ($selectAlias === $selectExpr && $fieldName === $selectField && isset($this->apiFieldSpec[$fieldName])) { + $field = $this->getField($fieldName); + FormattingUtil::formatInputValue($value, $expr, $field, $this->entityValues, $operator); + $fieldAlias = $selectAlias; + break; + } + } + } + if (!isset($fieldAlias)) { + if (in_array($expr, $this->getSelect())) { + throw new UnauthorizedException("Unauthorized field '$expr'"); + } + else { + throw new \CRM_Core_Exception("Invalid expression in HAVING clause: '$expr'. Must use a value from SELECT clause."); + } + } + $fieldAlias = '`' . $fieldAlias . '`'; + } + elseif ($type === 'ON' || ($type === 'WHERE' && $isExpression)) { + $expr = $this->getExpression($expr); + $fieldName = count($expr->getFields()) === 1 ? $expr->getFields()[0] : NULL; + $fieldAlias = $expr->render($this); + if (is_string($value)) { + $valExpr = $this->getExpression($value); + if ($expr->getType() === 'SqlField' && $valExpr->getType() === 'SqlString') { + $value = $valExpr->getExpr(); + FormattingUtil::formatInputValue($value, $fieldName, $this->apiFieldSpec[$fieldName], $this->entityValues, $operator); + return $this->createSQLClause($fieldAlias, $operator, $value, $this->apiFieldSpec[$fieldName], $depth); + } + else { + $value = $valExpr->render($this); + return sprintf('%s %s %s', $fieldAlias, $operator, $value); + } + } + elseif ($expr->getType() === 'SqlField') { + $field = $this->getField($fieldName); + FormattingUtil::formatInputValue($value, $fieldName, $field, $this->entityValues, $operator); + } + } + + $sqlClause = $this->createSQLClause($fieldAlias, $operator, $value, $field, $depth); + if ($sqlClause === NULL) { + throw new \CRM_Core_Exception("Invalid value in $type clause for '$expr'"); + } + return $sqlClause; + } + + /** + * @param string $fieldAlias + * @param string $operator + * @param mixed $value + * @param array|null $field + * @param int $depth + * @return array|string|NULL + * @throws \Exception + */ + protected function createSQLClause($fieldAlias, $operator, $value, $field, int $depth) { + if (!empty($field['operators']) && !in_array($operator, $field['operators'], TRUE)) { + throw new \CRM_Core_Exception('Illegal operator for ' . $field['name']); + } + // Some fields use a callback to generate their sql + if (!empty($field['sql_filters'])) { + $sql = []; + foreach ($field['sql_filters'] as $filter) { + $clause = is_callable($filter) ? $filter($field, $fieldAlias, $operator, $value, $this, $depth) : NULL; + if ($clause) { + $sql[] = $clause; + } + } + return $sql ? implode(' AND ', $sql) : NULL; + } + + // The CONTAINS and NOT CONTAINS operators match a substring for strings. + // For arrays & serialized fields, they only match a complete (not partial) string within the array. + if ($operator === 'CONTAINS' || $operator === 'NOT CONTAINS') { + $sep = \CRM_Core_DAO::VALUE_SEPARATOR; + switch ($field['serialize'] ?? NULL) { + + case \CRM_Core_DAO::SERIALIZE_JSON: + $operator = ($operator === 'CONTAINS') ? 'LIKE' : 'NOT LIKE'; + $value = '%"' . $value . '"%'; + // FIXME: Use this instead of the above hack once MIN_INSTALL_MYSQL_VER is bumped to 5.7. + // return sprintf('JSON_SEARCH(%s, "one", "%s") IS NOT NULL', $fieldAlias, \CRM_Core_DAO::escapeString($value)); + break; + + case \CRM_Core_DAO::SERIALIZE_SEPARATOR_BOOKEND: + $operator = ($operator === 'CONTAINS') ? 'LIKE' : 'NOT LIKE'; + // This is easy to query because the string is always bookended by separators. + $value = '%' . $sep . $value . $sep . '%'; + break; + + case \CRM_Core_DAO::SERIALIZE_SEPARATOR_TRIMMED: + $operator = ($operator === 'CONTAINS') ? 'REGEXP' : 'NOT REGEXP'; + // This is harder to query because there's no bookend. + // Use regex to match string within separators or content boundary + // Escaping regex per https://stackoverflow.com/questions/3782379/whats-the-best-way-to-escape-user-input-for-regular-expressions-in-mysql + $value = "(^|$sep)" . preg_quote($value, '&') . "($sep|$)"; + break; + + case \CRM_Core_DAO::SERIALIZE_COMMA: + $operator = ($operator === 'CONTAINS') ? 'REGEXP' : 'NOT REGEXP'; + // Match string within commas or content boundary + // Escaping regex per https://stackoverflow.com/questions/3782379/whats-the-best-way-to-escape-user-input-for-regular-expressions-in-mysql + $value = '(^|,)' . preg_quote($value, '&') . '(,|$)'; + break; + + default: + $operator = ($operator === 'CONTAINS') ? 'LIKE' : 'NOT LIKE'; + $value = '%' . $value . '%'; + break; + } + } + + if ($operator === 'IS EMPTY' || $operator === 'IS NOT EMPTY') { + // If field is not a string or number, this will pass through and use IS NULL/IS NOT NULL + $operator = str_replace('EMPTY', 'NULL', $operator); + // For strings & numbers, create an OR grouping of empty value OR null + if (in_array($field['data_type'] ?? NULL, ['String', 'Integer', 'Float'], TRUE)) { + $emptyVal = $field['data_type'] === 'String' ? '""' : '0'; + $isEmptyClause = $operator === 'IS NULL' ? "= $emptyVal OR" : "<> $emptyVal AND"; + return "($fieldAlias $isEmptyClause $fieldAlias $operator)"; + } + } + + if ($operator == 'REGEXP' || $operator == 'NOT REGEXP') { + return sprintf('%s %s "%s"', $fieldAlias, $operator, \CRM_Core_DAO::escapeString($value)); + } + + if (!$value && ($operator === 'IN' || $operator === 'NOT IN')) { + $value[] = FALSE; + } + + if (is_bool($value)) { + $value = (int) $value; + } + + return \CRM_Core_DAO::createSQLFilter($fieldAlias, [$operator => $value]); + } + + /** + * @return array + */ + public function getSelect() { + return $this->api->getSelect(); + } + + /** + * @return array + */ + public function getWhere() { + return $this->api->getWhere(); + } + + /** + * @return array + */ + public function getHaving() { + return $this->api->getHaving(); + } + + /** + * @return array + */ + public function getJoin() { + return $this->api->getJoin(); + } + + /** + * @return array + */ + public function getGroupBy() { + return $this->api->getGroupBy(); + } + + /** + * @return array + */ + public function getOrderBy() { + return $this->api->getOrderBy(); + } + + /** + * @return mixed + */ + public function getLimit() { + return $this->api->getLimit(); + } + + /** + * @return mixed + */ + public function getOffset() { + return $this->api->getOffset(); + } + + /** + * @return \CRM_Utils_SQL_Select + */ + public function getQuery() { + return $this->query; + } + + /** + * @return bool|string + */ + public function getCheckPermissions() { + return $this->api->getCheckPermissions(); + } + + /** + * Add something to the api's debug output if debugging is enabled + * + * @param $key + * @param $item + */ + public function debug($key, $item) { + if ($this->api->getDebug()) { + $this->api->_debugOutput[$key][] = $item; + } + } + +} diff --git a/Civi/Api4/Query/Api4SelectQuery.php b/Civi/Api4/Query/Api4SelectQuery.php index fa400f66f27d..aeb67216fb73 100644 --- a/Civi/Api4/Query/Api4SelectQuery.php +++ b/Civi/Api4/Query/Api4SelectQuery.php @@ -19,29 +19,9 @@ use Civi\Api4\Utils\SelectUtil; /** - * A query `node` may be in one of three formats: - * - * * leaf: [$fieldName, $operator, $criteria] - * * negated: ['NOT', $node] - * * branch: ['OR|NOT', [$node, $node, ...]] - * - * Leaf operators are one of: - * - * * '=', '<=', '>=', '>', '<', 'LIKE', "<>", "!=", - * * 'NOT LIKE', 'IN', 'NOT IN', 'BETWEEN', 'NOT BETWEEN', - * * 'IS NOT NULL', 'IS NULL', 'CONTAINS', 'NOT CONTAINS', - * * 'IS EMPTY', 'IS NOT EMPTY', 'REGEXP', 'NOT REGEXP'. + * Constructs SELECT FROM queries for API4 GET actions. */ -class Api4SelectQuery { - - const - MAIN_TABLE_ALIAS = 'a', - UNLIMITED = '18446744073709551615'; - - /** - * @var \CRM_Utils_SQL_Select - */ - protected $query; +class Api4SelectQuery extends Api4Query { /** * Used to keep track of implicit join table aliases @@ -55,27 +35,11 @@ class Api4SelectQuery { */ protected $autoJoinSuffix = 0; - /** - * @var array[] - */ - protected $apiFieldSpec; - /** * @var array */ protected $aclFields = []; - /** - * @var \Civi\Api4\Generic\DAOGetAction - */ - private $api; - - /** - * @var array - * [alias => expr][] - */ - protected $selectAliases = []; - /** * @var bool */ @@ -92,15 +56,10 @@ class Api4SelectQuery { private $entityAccess = []; /** - * @var array - */ - private $entityValues = []; - - /** - * @param \Civi\Api4\Generic\DAOGetAction $apiGet + * @param \Civi\Api4\Generic\DAOGetAction $api */ - public function __construct($apiGet) { - $this->api = $apiGet; + public function __construct($api) { + parent::__construct($api); // Always select ID of main table unless grouping by something else $keys = CoreUtil::getInfoItem($this->getEntity(), 'primary_key'); @@ -127,55 +86,16 @@ public function __construct($apiGet) { $this->addExplicitJoins(); } - protected function isAggregateQuery() { - if ($this->getGroupBy()) { - return TRUE; - } - foreach ($this->getSelect() as $sql) { - $classname = get_class(SqlExpression::convert($sql, TRUE)); - if (method_exists($classname, 'getCategory') && $classname::getCategory() === SqlFunction::CATEGORY_AGGREGATE) { - return TRUE; - } - } - return FALSE; - } - - /** - * Builds main final sql statement after initialization. - * - * @return string - * @throws \CRM_Core_Exception - */ - public function getSql() { - $this->buildSelectClause(); - $this->buildWhereClause(); - $this->buildOrderBy(); - $this->buildLimit(); - $this->buildGroupBy(); - $this->buildHavingClause(); - return $this->query->toSQL(); - } - /** * Why walk when you can * * @return array */ - public function run() { - $results = []; - $sql = $this->getSql(); - $this->debug('sql', $sql); - $query = \CRM_Core_DAO::executeQuery($sql); - while ($query->fetch()) { - $result = []; - foreach ($this->selectAliases as $alias => $expr) { - $returnName = $alias; - $alias = str_replace('.', '_', $alias); - $result[$returnName] = property_exists($query, $alias) ? $query->$alias : NULL; - } - $results[] = $result; + public function run(): array { + $results = $this->getResults(); + foreach ($results as &$result) { + FormattingUtil::formatOutputValues($result, $this->apiFieldSpec, 'get', $this->selectAliases); } - FormattingUtil::formatOutputValues($results, $this->apiFieldSpec, 'get', $this->selectAliases); return $results; } @@ -360,27 +280,6 @@ protected function buildOrderBy() { } } - /** - * Add LIMIT to query - * - * @throws \CRM_Core_Exception - */ - protected function buildLimit() { - if ($this->getLimit() || $this->getOffset()) { - // If limit is 0, mysql will actually return 0 results. Instead set to maximum possible. - $this->query->limit($this->getLimit() ?: self::UNLIMITED, $this->getOffset()); - } - } - - /** - * Add GROUP BY clause to query - */ - protected function buildGroupBy() { - foreach ($this->getGroupBy() as $item) { - $this->query->groupBy($this->renderExpr($this->getExpression($item))); - } - } - /** * This takes all the where clauses that use `=` to build an array of known values which every record must have. * @@ -414,266 +313,6 @@ private function fillEntityValues() { } } - /** - * Recursively validate and transform a branch or leaf clause array to SQL. - * - * @param array $clause - * @param string $type - * WHERE|HAVING|ON - * @param int $depth - * @return string SQL where clause - * - * @throws \CRM_Core_Exception - * @uses composeClause() to generate the SQL etc. - */ - protected function treeWalkClauses($clause, $type, $depth = 0) { - // Skip empty leaf. - if (in_array($clause[0], ['AND', 'OR', 'NOT']) && empty($clause[1])) { - return ''; - } - switch ($clause[0]) { - case 'OR': - case 'AND': - // handle branches - if (count($clause[1]) === 1) { - // a single set so AND|OR is immaterial - return $this->treeWalkClauses($clause[1][0], $type, $depth + 1); - } - else { - $sql_subclauses = []; - foreach ($clause[1] as $subclause) { - $sql_subclauses[] = $this->treeWalkClauses($subclause, $type, $depth + 1); - } - return '(' . implode("\n" . $clause[0] . ' ', $sql_subclauses) . ')'; - } - - case 'NOT': - // If we get a group of clauses with no operator, assume AND - if (!is_string($clause[1][0])) { - $clause[1] = ['AND', $clause[1]]; - } - return 'NOT (' . $this->treeWalkClauses($clause[1], $type, $depth + 1) . ')'; - - default: - try { - return $this->composeClause($clause, $type, $depth); - } - // Silently ignore fields the user lacks permission to see - catch (UnauthorizedException $e) { - return ''; - } - } - } - - /** - * Validate and transform a leaf clause array to SQL. - * @param array $clause [$fieldName, $operator, $criteria, $isExpression] - * @param string $type - * WHERE|HAVING|ON - * @param int $depth - * @return string SQL - * @throws \CRM_Core_Exception - * @throws \Exception - */ - public function composeClause(array $clause, string $type, int $depth) { - $field = NULL; - // Pad array for unary operators - [$expr, $operator, $value] = array_pad($clause, 3, NULL); - $isExpression = $clause[3] ?? FALSE; - if (!in_array($operator, CoreUtil::getOperators(), TRUE)) { - throw new \CRM_Core_Exception('Illegal operator'); - } - - // For WHERE clause, expr must be the name of a field. - if ($type === 'WHERE' && !$isExpression) { - $expr = $this->getExpression($expr, ['SqlField', 'SqlFunction', 'SqlEquation']); - if ($expr->getType() === 'SqlField') { - $fieldName = count($expr->getFields()) === 1 ? $expr->getFields()[0] : NULL; - $field = $this->getField($fieldName, TRUE); - FormattingUtil::formatInputValue($value, $fieldName, $field, $this->entityValues, $operator); - } - elseif ($expr->getType() === 'SqlFunction') { - $fauxField = [ - 'name' => NULL, - 'data_type' => $expr::getDataType(), - ]; - FormattingUtil::formatInputValue($value, NULL, $fauxField, $this->entityValues, $operator); - } - $fieldAlias = $expr->render($this); - } - // For HAVING, expr must be an item in the SELECT clause - elseif ($type === 'HAVING') { - // Expr references a fieldName or alias - if (isset($this->selectAliases[$expr])) { - $fieldAlias = $expr; - // Attempt to format if this is a real field - if (isset($this->apiFieldSpec[$expr])) { - $field = $this->getField($expr); - FormattingUtil::formatInputValue($value, $expr, $field, $this->entityValues, $operator); - } - } - // Expr references a non-field expression like a function; convert to alias - elseif (in_array($expr, $this->selectAliases)) { - $fieldAlias = array_search($expr, $this->selectAliases); - } - // If either the having or select field contains a pseudoconstant suffix, match and perform substitution - else { - [$fieldName] = explode(':', $expr); - foreach ($this->selectAliases as $selectAlias => $selectExpr) { - [$selectField] = explode(':', $selectAlias); - if ($selectAlias === $selectExpr && $fieldName === $selectField && isset($this->apiFieldSpec[$fieldName])) { - $field = $this->getField($fieldName); - FormattingUtil::formatInputValue($value, $expr, $field, $this->entityValues, $operator); - $fieldAlias = $selectAlias; - break; - } - } - } - if (!isset($fieldAlias)) { - if (in_array($expr, $this->getSelect())) { - throw new UnauthorizedException("Unauthorized field '$expr'"); - } - else { - throw new \CRM_Core_Exception("Invalid expression in HAVING clause: '$expr'. Must use a value from SELECT clause."); - } - } - $fieldAlias = '`' . $fieldAlias . '`'; - } - elseif ($type === 'ON' || ($type === 'WHERE' && $isExpression)) { - $expr = $this->getExpression($expr); - $fieldName = count($expr->getFields()) === 1 ? $expr->getFields()[0] : NULL; - $fieldAlias = $expr->render($this); - if (is_string($value)) { - $valExpr = $this->getExpression($value); - if ($expr->getType() === 'SqlField' && $valExpr->getType() === 'SqlString') { - $value = $valExpr->getExpr(); - FormattingUtil::formatInputValue($value, $fieldName, $this->apiFieldSpec[$fieldName], $this->entityValues, $operator); - return $this->createSQLClause($fieldAlias, $operator, $value, $this->apiFieldSpec[$fieldName], $depth); - } - else { - $value = $valExpr->render($this); - return sprintf('%s %s %s', $fieldAlias, $operator, $value); - } - } - elseif ($expr->getType() === 'SqlField') { - $field = $this->getField($fieldName); - FormattingUtil::formatInputValue($value, $fieldName, $field, $this->entityValues, $operator); - } - } - - $sqlClause = $this->createSQLClause($fieldAlias, $operator, $value, $field, $depth); - if ($sqlClause === NULL) { - throw new \CRM_Core_Exception("Invalid value in $type clause for '$expr'"); - } - return $sqlClause; - } - - /** - * @param string $fieldAlias - * @param string $operator - * @param mixed $value - * @param array|null $field - * @param int $depth - * @return array|string|NULL - * @throws \Exception - */ - protected function createSQLClause($fieldAlias, $operator, $value, $field, int $depth) { - if (!empty($field['operators']) && !in_array($operator, $field['operators'], TRUE)) { - throw new \CRM_Core_Exception('Illegal operator for ' . $field['name']); - } - // Some fields use a callback to generate their sql - if (!empty($field['sql_filters'])) { - $sql = []; - foreach ($field['sql_filters'] as $filter) { - $clause = is_callable($filter) ? $filter($field, $fieldAlias, $operator, $value, $this, $depth) : NULL; - if ($clause) { - $sql[] = $clause; - } - } - return $sql ? implode(' AND ', $sql) : NULL; - } - - // The CONTAINS and NOT CONTAINS operators match a substring for strings. - // For arrays & serialized fields, they only match a complete (not partial) string within the array. - if ($operator === 'CONTAINS' || $operator === 'NOT CONTAINS') { - $sep = \CRM_Core_DAO::VALUE_SEPARATOR; - switch ($field['serialize'] ?? NULL) { - - case \CRM_Core_DAO::SERIALIZE_JSON: - $operator = ($operator === 'CONTAINS') ? 'LIKE' : 'NOT LIKE'; - $value = '%"' . $value . '"%'; - // FIXME: Use this instead of the above hack once MIN_INSTALL_MYSQL_VER is bumped to 5.7. - // return sprintf('JSON_SEARCH(%s, "one", "%s") IS NOT NULL', $fieldAlias, \CRM_Core_DAO::escapeString($value)); - break; - - case \CRM_Core_DAO::SERIALIZE_SEPARATOR_BOOKEND: - $operator = ($operator === 'CONTAINS') ? 'LIKE' : 'NOT LIKE'; - // This is easy to query because the string is always bookended by separators. - $value = '%' . $sep . $value . $sep . '%'; - break; - - case \CRM_Core_DAO::SERIALIZE_SEPARATOR_TRIMMED: - $operator = ($operator === 'CONTAINS') ? 'REGEXP' : 'NOT REGEXP'; - // This is harder to query because there's no bookend. - // Use regex to match string within separators or content boundary - // Escaping regex per https://stackoverflow.com/questions/3782379/whats-the-best-way-to-escape-user-input-for-regular-expressions-in-mysql - $value = "(^|$sep)" . preg_quote($value, '&') . "($sep|$)"; - break; - - case \CRM_Core_DAO::SERIALIZE_COMMA: - $operator = ($operator === 'CONTAINS') ? 'REGEXP' : 'NOT REGEXP'; - // Match string within commas or content boundary - // Escaping regex per https://stackoverflow.com/questions/3782379/whats-the-best-way-to-escape-user-input-for-regular-expressions-in-mysql - $value = '(^|,)' . preg_quote($value, '&') . '(,|$)'; - break; - - default: - $operator = ($operator === 'CONTAINS') ? 'LIKE' : 'NOT LIKE'; - $value = '%' . $value . '%'; - break; - } - } - - if ($operator === 'IS EMPTY' || $operator === 'IS NOT EMPTY') { - // If field is not a string or number, this will pass through and use IS NULL/IS NOT NULL - $operator = str_replace('EMPTY', 'NULL', $operator); - // For strings & numbers, create an OR grouping of empty value OR null - if (in_array($field['data_type'] ?? NULL, ['String', 'Integer', 'Float'], TRUE)) { - $emptyVal = $field['data_type'] === 'String' ? '""' : '0'; - $isEmptyClause = $operator === 'IS NULL' ? "= $emptyVal OR" : "<> $emptyVal AND"; - return "($fieldAlias $isEmptyClause $fieldAlias $operator)"; - } - } - - if ($operator == 'REGEXP' || $operator == 'NOT REGEXP') { - return sprintf('%s %s "%s"', $fieldAlias, $operator, \CRM_Core_DAO::escapeString($value)); - } - - if (!$value && ($operator === 'IN' || $operator === 'NOT IN')) { - $value[] = FALSE; - } - - if (is_bool($value)) { - $value = (int) $value; - } - - return \CRM_Core_DAO::createSQLFilter($fieldAlias, [$operator => $value]); - } - - /** - * @param string $expr - * @param array $allowedTypes - * @return SqlExpression - * @throws \CRM_Core_Exception - */ - protected function getExpression(string $expr, $allowedTypes = NULL) { - $sqlExpr = SqlExpression::convert($expr, FALSE, $allowedTypes); - foreach ($sqlExpr->getFields() as $fieldName) { - $this->getField($fieldName, TRUE); - } - return $sqlExpr; - } - /** * Get acl clause for an entity * @@ -1213,76 +852,6 @@ public function getEntity() { return $this->api->getEntityName(); } - /** - * @return array - */ - public function getSelect() { - return $this->api->getSelect(); - } - - /** - * @return array - */ - public function getWhere() { - return $this->api->getWhere(); - } - - /** - * @return array - */ - public function getHaving() { - return $this->api->getHaving(); - } - - /** - * @return array - */ - public function getJoin() { - return $this->api->getJoin(); - } - - /** - * @return array - */ - public function getGroupBy() { - return $this->api->getGroupBy(); - } - - /** - * @return array - */ - public function getOrderBy() { - return $this->api->getOrderBy(); - } - - /** - * @return mixed - */ - public function getLimit() { - return $this->api->getLimit(); - } - - /** - * @return mixed - */ - public function getOffset() { - return $this->api->getOffset(); - } - - /** - * @return \CRM_Utils_SQL_Select - */ - public function getQuery() { - return $this->query; - } - - /** - * @return bool|string - */ - public function getCheckPermissions() { - return $this->api->getCheckPermissions(); - } - /** * @param string $alias * @return array{entity: string, alias: string, table: string, bridge: string|NULL}|NULL @@ -1298,22 +867,6 @@ public function getExplicitJoins() { return $this->explicitJoins; } - /** - * @param string $path - * @param array $field - */ - private function addSpecField($path, $field) { - // Only add field to spec if we have permission - if ($this->getCheckPermissions() && !empty($field['permission']) && !\CRM_Core_Permission::check($field['permission'])) { - $this->apiFieldSpec[$path] = FALSE; - return; - } - $this->apiFieldSpec[$path] = $field + [ - 'implicit_join' => NULL, - 'explicit_join' => NULL, - ]; - } - /** * Returns rendered expression or alias if it is already aliased in the SELECT clause. * @@ -1333,16 +886,4 @@ protected function renderExpr($expr) { return $expr->render($this); } - /** - * Add something to the api's debug output if debugging is enabled - * - * @param $key - * @param $item - */ - public function debug($key, $item) { - if ($this->api->getDebug()) { - $this->api->_debugOutput[$key][] = $item; - } - } - } diff --git a/Civi/Api4/Query/SqlEquation.php b/Civi/Api4/Query/SqlEquation.php index da5e9d2f1cd0..a6dd5f564f18 100644 --- a/Civi/Api4/Query/SqlEquation.php +++ b/Civi/Api4/Query/SqlEquation.php @@ -76,10 +76,10 @@ public function getArgs(): array { /** * Render the expression for insertion into the sql query * - * @param \Civi\Api4\Query\Api4SelectQuery $query + * @param \Civi\Api4\Query\Api4Query $query * @return string */ - public function render(Api4SelectQuery $query): string { + public function render(Api4Query $query): string { $output = []; foreach ($this->args as $i => $arg) { // Just an operator diff --git a/Civi/Api4/Query/SqlExpression.php b/Civi/Api4/Query/SqlExpression.php index f4192477d398..ad1109663ae8 100644 --- a/Civi/Api4/Query/SqlExpression.php +++ b/Civi/Api4/Query/SqlExpression.php @@ -140,10 +140,10 @@ public function getFields(): array { /** * Renders expression to a sql string, replacing field names with column names. * - * @param \Civi\Api4\Query\Api4SelectQuery $query + * @param \Civi\Api4\Query\Api4Query $query * @return string */ - abstract public function render(Api4SelectQuery $query): string; + abstract public function render(Api4Query $query): string; /** * @return string diff --git a/Civi/Api4/Query/SqlField.php b/Civi/Api4/Query/SqlField.php index a921dabfaa02..862f67aea087 100644 --- a/Civi/Api4/Query/SqlField.php +++ b/Civi/Api4/Query/SqlField.php @@ -19,13 +19,13 @@ class SqlField extends SqlExpression { public $supportsExpansion = TRUE; protected function initialize() { - if ($this->alias && $this->alias !== $this->expr) { + if ($this->alias && $this->alias !== $this->expr && !strpos($this->expr, ':')) { throw new \CRM_Core_Exception("Aliasing field names is not allowed, only expressions can have an alias."); } $this->fields[] = $this->expr; } - public function render(Api4SelectQuery $query): string { + public function render(Api4Query $query): string { $field = $query->getField($this->expr, TRUE); if (!empty($field['sql_renderer'])) { $renderer = $field['sql_renderer']; diff --git a/Civi/Api4/Query/SqlFunction.php b/Civi/Api4/Query/SqlFunction.php index ffb9506e5280..a766e95bbdb2 100644 --- a/Civi/Api4/Query/SqlFunction.php +++ b/Civi/Api4/Query/SqlFunction.php @@ -142,10 +142,10 @@ public function formatOutputValue($value, &$dataType) { /** * Render the expression for insertion into the sql query * - * @param \Civi\Api4\Query\Api4SelectQuery $query + * @param \Civi\Api4\Query\Api4Query $query * @return string */ - public function render(Api4SelectQuery $query): string { + public function render(Api4Query $query): string { $output = ''; foreach ($this->args as $arg) { $rendered = $this->renderArg($arg, $query); @@ -168,10 +168,10 @@ protected function renderExpression($output): string { /** * @param array $arg - * @param \Civi\Api4\Query\Api4SelectQuery $query + * @param \Civi\Api4\Query\Api4Query $query * @return string */ - private function renderArg($arg, Api4SelectQuery $query): string { + private function renderArg($arg, Api4Query $query): string { $rendered = implode(' ', $arg['prefix']); foreach ($arg['expr'] ?? [] as $idx => $expr) { if (strlen($rendered) || $idx) { diff --git a/Civi/Api4/Query/SqlNull.php b/Civi/Api4/Query/SqlNull.php index a56720f3d5e4..312f01fa4b55 100644 --- a/Civi/Api4/Query/SqlNull.php +++ b/Civi/Api4/Query/SqlNull.php @@ -19,7 +19,7 @@ class SqlNull extends SqlExpression { protected function initialize() { } - public function render(Api4SelectQuery $query): string { + public function render(Api4Query $query): string { return 'NULL'; } diff --git a/Civi/Api4/Query/SqlNumber.php b/Civi/Api4/Query/SqlNumber.php index 14025f707cde..82c784a845d2 100644 --- a/Civi/Api4/Query/SqlNumber.php +++ b/Civi/Api4/Query/SqlNumber.php @@ -22,7 +22,7 @@ protected function initialize() { \CRM_Utils_Type::validate($this->expr, 'Float'); } - public function render(Api4SelectQuery $query): string { + public function render(Api4Query $query): string { return $this->expr; } diff --git a/Civi/Api4/Query/SqlString.php b/Civi/Api4/Query/SqlString.php index 51b5422c9850..3c64533251a9 100644 --- a/Civi/Api4/Query/SqlString.php +++ b/Civi/Api4/Query/SqlString.php @@ -27,7 +27,7 @@ protected function initialize() { $this->expr = str_replace(['\\\\', "\\$quot", $backslash], [$backslash, $quot, '\\\\'], $str); } - public function render(Api4SelectQuery $query): string { + public function render(Api4Query $query): string { return '"' . \CRM_Core_DAO::escapeString($this->expr) . '"'; } diff --git a/Civi/Api4/Query/SqlWild.php b/Civi/Api4/Query/SqlWild.php index 090f864f5ff8..972c47f1d8d6 100644 --- a/Civi/Api4/Query/SqlWild.php +++ b/Civi/Api4/Query/SqlWild.php @@ -19,7 +19,7 @@ class SqlWild extends SqlExpression { protected function initialize() { } - public function render(Api4SelectQuery $query): string { + public function render(Api4Query $query): string { return '*'; } diff --git a/Civi/Api4/Utils/FormattingUtil.php b/Civi/Api4/Utils/FormattingUtil.php index d91138107ccc..522833a570a3 100644 --- a/Civi/Api4/Utils/FormattingUtil.php +++ b/Civi/Api4/Utils/FormattingUtil.php @@ -213,56 +213,54 @@ public static function formatDateValue($format, $value, &$operator = NULL, $inde /** * Unserialize raw DAO values and convert to correct type * - * @param array $results + * @param array $result * @param array $fields * @param string $action * @param array $selectAliases * @throws \CRM_Core_Exception */ - public static function formatOutputValues(&$results, $fields, $action = 'get', $selectAliases = []) { - foreach ($results as &$result) { - $contactTypePaths = []; - foreach ($result as $key => $value) { - $fieldExpr = SqlExpression::convert($selectAliases[$key] ?? $key); - $fieldName = \CRM_Utils_Array::first($fieldExpr->getFields() ?? ''); - $baseName = $fieldName ? \CRM_Utils_Array::first(explode(':', $fieldName)) : NULL; - $field = $fields[$fieldName] ?? $fields[$baseName] ?? NULL; - $dataType = $field['data_type'] ?? ($fieldName == 'id' ? 'Integer' : NULL); - // Allow Sql Functions to do special formatting and/or alter the $dataType - if (method_exists($fieldExpr, 'formatOutputValue') && is_string($value)) { - $result[$key] = $value = $fieldExpr->formatOutputValue($value, $dataType); - } - if (!empty($field['output_formatters'])) { - self::applyFormatters($result, $fieldName, $field, $value); - $dataType = NULL; - } - // Evaluate pseudoconstant suffixes - $suffix = strrpos(($fieldName ?? ''), ':'); - $fieldOptions = NULL; - if (isset($value) && $suffix) { - $fieldOptions = self::getPseudoconstantList($field, $fieldName, $result, $action); - $dataType = NULL; - } - // Store contact_type value before replacing pseudoconstant (e.g. transforming it to contact_type:label) - // Used by self::contactFieldsToRemove below - if ($value && isset($field['entity']) && $field['entity'] === 'Contact' && $field['name'] === 'contact_type') { - $prefix = strrpos($fieldName, '.'); - $contactTypePaths[$prefix ? substr($fieldName, 0, $prefix + 1) : ''] = $value; + public static function formatOutputValues(&$result, $fields, $action = 'get', $selectAliases = []) { + $contactTypePaths = []; + foreach ($result as $key => $value) { + $fieldExpr = SqlExpression::convert($selectAliases[$key] ?? $key); + $fieldName = \CRM_Utils_Array::first($fieldExpr->getFields() ?? ''); + $baseName = $fieldName ? \CRM_Utils_Array::first(explode(':', $fieldName)) : NULL; + $field = $fields[$fieldName] ?? $fields[$baseName] ?? NULL; + $dataType = $field['data_type'] ?? ($fieldName == 'id' ? 'Integer' : NULL); + // Allow Sql Functions to do special formatting and/or alter the $dataType + if (method_exists($fieldExpr, 'formatOutputValue') && is_string($value)) { + $result[$key] = $value = $fieldExpr->formatOutputValue($value, $dataType); + } + if (!empty($field['output_formatters'])) { + self::applyFormatters($result, $fieldName, $field, $value); + $dataType = NULL; + } + // Evaluate pseudoconstant suffixes + $suffix = strrpos(($fieldName ?? ''), ':'); + $fieldOptions = NULL; + if (isset($value) && $suffix) { + $fieldOptions = self::getPseudoconstantList($field, $fieldName, $result, $action); + $dataType = NULL; + } + // Store contact_type value before replacing pseudoconstant (e.g. transforming it to contact_type:label) + // Used by self::contactFieldsToRemove below + if ($value && isset($field['entity']) && $field['entity'] === 'Contact' && $field['name'] === 'contact_type') { + $prefix = strrpos($fieldName, '.'); + $contactTypePaths[$prefix ? substr($fieldName, 0, $prefix + 1) : ''] = $value; + } + if ($fieldExpr->supportsExpansion) { + if (!empty($field['serialize']) && is_string($value)) { + $value = \CRM_Core_DAO::unSerializeField($value, $field['serialize']); } - if ($fieldExpr->supportsExpansion) { - if (!empty($field['serialize']) && is_string($value)) { - $value = \CRM_Core_DAO::unSerializeField($value, $field['serialize']); - } - if (isset($fieldOptions)) { - $value = self::replacePseudoconstant($fieldOptions, $value); - } + if (isset($fieldOptions)) { + $value = self::replacePseudoconstant($fieldOptions, $value); } - $result[$key] = self::convertDataType($value, $dataType); - } - // Remove inapplicable contact fields - foreach ($contactTypePaths as $prefix => $contactType) { - \CRM_Utils_Array::remove($result, self::contactFieldsToRemove($contactType, $prefix)); } + $result[$key] = self::convertDataType($value, $dataType); + } + // Remove inapplicable contact fields + foreach ($contactTypePaths as $prefix => $contactType) { + \CRM_Utils_Array::remove($result, self::contactFieldsToRemove($contactType, $prefix)); } } diff --git a/ext/search_kit/Civi/Api4/Action/SearchDisplay/GetDefault.php b/ext/search_kit/Civi/Api4/Action/SearchDisplay/GetDefault.php index b21aabe9cba6..d4939e4d7fad 100644 --- a/ext/search_kit/Civi/Api4/Action/SearchDisplay/GetDefault.php +++ b/ext/search_kit/Civi/Api4/Action/SearchDisplay/GetDefault.php @@ -91,10 +91,9 @@ public function _run(\Civi\Api4\Generic\Result $result) { $display[$fieldExpr] = $display[$fieldName]; } } - $results = [$display]; // Replace pseudoconstants e.g. type:icon - FormattingUtil::formatOutputValues($results, $fields); - $result->exchangeArray($this->selectArray($results)); + FormattingUtil::formatOutputValues($display, $fields); + $result->exchangeArray($this->selectArray([$display])); } /** diff --git a/tests/phpunit/api/v4/Action/EntitySetUnionTest.php b/tests/phpunit/api/v4/Action/EntitySetUnionTest.php new file mode 100644 index 000000000000..0ccfca035d4c --- /dev/null +++ b/tests/phpunit/api/v4/Action/EntitySetUnionTest.php @@ -0,0 +1,120 @@ +saveTestRecords('Group', [ + 'records' => [ + ['title' => '1G', 'description' => 'Group 1'], + ['title' => '2G', 'description' => 'Group 2'], + ['title' => '3G', 'group_type:name' => ['Access Control', 'Mailing List']], + ], + ]); + $this->saveTestRecords('Tag', [ + 'records' => [ + ['name' => '3T', 'description' => 'Tag 3', 'used_for:name' => ['Contact', 'Activity']], + ['name' => '2T', 'description' => 'Tag 2'], + ['name' => '1T', 'description' => 'Tag 1'], + ], + ]); + $result = EntitySet::get(FALSE) + ->addSet('UNION ALL', Group::get() + ->addSelect('title', 'description', '"group" AS thing') + ->addWhere('title', 'IN', ['1G', '2G', '3G']) + ) + ->addSet('UNION ALL', Tag::get() + // The UNION will automatically alias Tag."name" to "title" because that's the column name in the 1st query + ->addSelect('name', 'description', '"tag" AS thing') + ->addWhere('name', 'IN', ['1T', '2T', '3T']) + ) + ->addOrderBy('title') + ->setLimit(5) + ->execute(); + + $this->assertCount(5, $result); + $this->assertEquals(['title' => '1G', 'description' => 'Group 1', 'thing' => 'group'], $result[0]); + $this->assertEquals(['title' => '1T', 'description' => 'Tag 1', 'thing' => 'tag'], $result[1]); + $this->assertEquals(['title' => '2G', 'description' => 'Group 2', 'thing' => 'group'], $result[2]); + $this->assertEquals(['title' => '2T', 'description' => 'Tag 2', 'thing' => 'tag'], $result[3]); + $this->assertEquals(['title' => '3G', 'description' => NULL, 'thing' => 'group'], $result[4]); + + // Try with a "WHERE" clause + $result = EntitySet::get(FALSE) + ->addSet('UNION ALL', Group::get() + ->addSelect('title', 'description', 'group_type:name AS type') + ->addWhere('title', 'IN', ['1G', '2G', '3G']) + ) + ->addSet('UNION ALL', Tag::get() + ->addSelect('name', 'description', 'used_for:name') + ->addWhere('name', 'IN', ['1T', '2T', '3T']) + ) + ->addOrderBy('title') + ->addWhere('title', 'LIKE', '3%') + ->setDebug(TRUE) + ->execute(); + $this->assertCount(2, $result); + // Correct pseudoconstants should have been looked up for each row + $this->assertEquals(['Access Control', 'Mailing List'], $result[0]['type']); + $this->assertEquals(['Contact', 'Activity'], $result[1]['type']); + } + + public function testGroupByUnionSet(): void { + $contacts = $this->saveTestRecords('Contact', ['records' => 4])->column('id'); + $relationships = $this->saveTestRecords('Relationship', [ + 'records' => [ + ['contact_id_a' => $contacts[0], 'contact_id_b' => $contacts[1]], + ['contact_id_a' => $contacts[1], 'contact_id_b' => $contacts[2]], + ['contact_id_a' => $contacts[2], 'contact_id_b' => $contacts[3]], + ], + ]); + $result = EntitySet::get(FALSE) + ->addSelect('COUNT(id) AS count', 'contact_id_a') + ->addSet('UNION ALL', Relationship::get() + ->addSelect('id', 'contact_id_a', 'contact_id_b', '"a_b" AS direction') + ->addWhere('id', 'IN', $relationships->column('id')) + ) + ->addSet('UNION ALL', Relationship::get() + ->addSelect('id', 'contact_id_b', 'contact_id_a', '"b_a" AS direction') + ->addWhere('id', 'IN', $relationships->column('id')) + ) + ->addGroupBy('contact_id_a') + ->addOrderBy('contact_id_a') + ->execute(); + $this->assertCount(4, $result); + $this->assertEquals(1, $result[0]['count']); + $this->assertEquals(2, $result[1]['count']); + $this->assertEquals(2, $result[2]['count']); + $this->assertEquals(1, $result[3]['count']); + + } + +}