Skip to content

Commit

Permalink
FIX Resolve problems with eagerloading performance
Browse files Browse the repository at this point in the history
  • Loading branch information
@GuySartorelli
GuySartorelli committed Jul 5, 2023
1 parent 85e503d commit c29d7cf
Showing 1 changed file with 102 additions and 82 deletions.
184 changes: 102 additions & 82 deletions src/ORM/DataList.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1072,13 +1072,10 @@ private function fetchEagerLoadRelations(Query $query): void
if (empty($this->eagerLoadRelations)) {
return;
}
$ids = $query->column('ID');
if (empty($ids)) {
$topLevelIDs = $query->column('ID');
if (empty($topLevelIDs)) {
return;
}
$topLevelIDs = $ids;
// Using ->toArray() and then iterating instead of just iterating DataList because
// in some instances it prevents some extra SQL queries
$prevRelationArray = [];
foreach ($this->eagerLoadRelations as $eagerLoadRelation) {
list(
Expand All @@ -1089,52 +1086,52 @@ private function fetchEagerLoadRelations(Query $query): void
$hasManyIDField,
$manyManyLastComponent
) = $this->getEagerLoadVariables($eagerLoadRelation);
$dataClass = $dataClasses[count($dataClasses) - 2];
$relation = $relations[count($relations) - 1];
$parentDataClass = $dataClasses[count($dataClasses) - 2];
$relationName = $relations[count($relations) - 1];
$relationDataClass = $dataClasses[count($dataClasses) - 1];
if ($dataClass === $this->dataClass) {
if ($parentDataClass === $this->dataClass()) {
// When we're at "the top of a tree of nested relationships", we can just use the IDs from the query
// This is important to do when handling multiple eager-loaded relatioship trees.
$ids = $topLevelIDs;
// This is important to do when handling multiple eager-loaded relationship trees.
$parentIDs = $topLevelIDs;
}
// has_one
if ($hasOneIDField) {
list($prevRelationArray, $ids) = $this->fetchEagerLoadHasOne(
list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadHasOne(
$query,
$prevRelationArray,
$hasOneIDField,
$relationDataClass,
$eagerLoadRelation,
$relation,
$dataClass,
$dataClasses
$relationName,
$parentDataClass
);
// belongs_to
} elseif ($belongsToIDField) {
list($prevRelationArray, $ids) = $this->fetchEagerLoadBelongsTo(
$ids,
list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadBelongsTo(
$parentIDs,
$belongsToIDField,
$relationDataClass,
$eagerLoadRelation,
$relation
$relationName
);
// has_many
} elseif ($hasManyIDField) {
list($prevRelationArray, $ids) = $this->fetchEagerLoadHasMany(
$ids,
list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadHasMany(
$parentIDs,
$hasManyIDField,
$relationDataClass,
$eagerLoadRelation,
$relation
$relationName
);
// many_many + belongs_many_many & many_many_through
} elseif ($manyManyLastComponent) {
list($prevRelationArray, $ids) = $this->fetchEagerLoadManyMany(
list($prevRelationArray, $parentIDs) = $this->fetchEagerLoadManyMany(
$manyManyLastComponent,
$ids,
$parentIDs,
$relationDataClass,
$eagerLoadRelation,
$relation
$relationName,
$parentDataClass
);
} else {
throw new LogicException('Something went wrong with the eager loading');
Expand All @@ -1144,139 +1141,161 @@ private function fetchEagerLoadRelations(Query $query): void

private function fetchEagerLoadHasOne(
Query $query,
array $prevRelationArray,
array $parentRecords,
string $hasOneIDField,
string $relationDataClass,
string $eagerLoadRelation,
string $relation,
string $dataClass,
array $dataClasses
string $relationName,
string $parentDataClass
): array
{
$itemArray = [];
$relationItemIDs = [];
if ($dataClass === $dataClasses[0]) {
while ($row = $query->record()) {

// It's a has_one directly on the records in THIS list
if ($parentDataClass === $this->dataClass()) {
foreach ($query as $itemData) {
$itemArray[] = [
'ID' => $row['ID'],
$hasOneIDField => $row[$hasOneIDField]
'ID' => $itemData['ID'],
$hasOneIDField => $itemData[$hasOneIDField]
];
$relationItemIDs[] = $row[$hasOneIDField];
$relationItemIDs[] = $itemData[$hasOneIDField];
}
//
} else {
foreach ($prevRelationArray as $itemData) {
foreach ($parentRecords as $itemData) {
$itemArray[] = [
'ID' => $itemData->ID,
$hasOneIDField => $itemData->$hasOneIDField
];
$relationItemIDs[] = $itemData->$hasOneIDField;
}
}
$relationArray = DataObject::get($relationDataClass)->filter(['ID' => $relationItemIDs])->toArray();
$relationArray = DataObject::get($relationDataClass)->byIDs($relationItemIDs)->toArray();
foreach ($itemArray as $itemData) {
foreach ($relationArray as $relationItem) {
$eagerLoadID = $itemData['ID'];
if ($relationItem->ID === $itemData[$hasOneIDField]) {
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation] = $relationItem;
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName] = $relationItem;
}
}
}
return [$relationArray, $relationItemIDs];
}

private function fetchEagerLoadBelongsTo(
array $ids,
array $parentIDs,
string $belongsToIDField,
string $relationDataClass,
string $eagerLoadRelation,
string $relation
string $relationName
): array
{
$relationArray = DataObject::get($relationDataClass)->filter([$belongsToIDField => $ids])->toArray();
// Get ALL of the items for this relation up front, for ALL of the parents
// Fetched as an array to avoid sporadic additional queries when the DataList is looped directly
$relationArray = DataObject::get($relationDataClass)->filter([$belongsToIDField => $parentIDs])->toArray();
$relationItemIDs = [];

// Store the children against the correct parent
foreach ($relationArray as $relationItem) {
$relationItemIDs[] = $relationItem->ID;
$eagerLoadID = $relationItem->$belongsToIDField;
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation] = $relationItem;
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName] = $relationItem;
}

return [$relationArray, $relationItemIDs];
}

private function fetchEagerLoadHasMany(
array $ids,
array $parentIDs,
string $hasManyIDField,
string $relationDataClass,
string $eagerLoadRelation,
string $relation
string $relationName
): array
{
$relationArray = DataObject::get($relationDataClass)->filter([$hasManyIDField => $ids])->toArray();
// Get ALL of the items for this relation up front, for ALL of the parents
// Fetched as an array to avoid sporadic additional queries when the DataList is looped directly
$relationArray = DataObject::get($relationDataClass)->filter([$hasManyIDField => $parentIDs])->toArray();
$relationItemIDs = [];

// Store the children in an ArrayList against the correct parent
foreach ($relationArray as $relationItem) {
$relationItemIDs[] = $relationItem->ID;
$eagerLoadID = $relationItem->$hasManyIDField;
if (!isset($this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation])) {
if (!isset($this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName])) {
$arrayList = ArrayList::create();
$arrayList->setDataClass($relationItem->dataClass);
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation] = $arrayList;
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName] = $arrayList;
}
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation]->push($relationItem);
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relationName]->push($relationItem);
}

return [$relationArray, $relationItemIDs];
}

private function fetchEagerLoadManyMany(
array $manyManyLastComponent,
array $ids,
array $parentIDs,
string $relationDataClass,
string $eagerLoadRelation,
string $relation
string $relationName,
string $parentDataClass
): array
{
$parentField = $manyManyLastComponent['parentField'];
$childField = $manyManyLastComponent['childField'];
$parentIDField = $manyManyLastComponent['parentField'];
$childIDField = $manyManyLastComponent['childField'];
// $join will either be:
// - the join table name for many-many
// - the join data class for many-many-through
$join = $manyManyLastComponent['join'];

// many_many_through
if (is_a($manyManyLastComponent['relationClass'], ManyManyThroughList::class, true)) {
$joinThroughObjs = $join::get()->filter([$parentField => $ids]);
$joinThroughObjs = DataObject::get($join)->filter([$parentIDField => $parentIDs]);
$relationItemIDs = [];
$rows = [];
$joinRows = [];
foreach ($joinThroughObjs as $joinThroughObj) {
$rows[] = [
$parentField => $joinThroughObj->$parentField,
$childField => $joinThroughObj->$childField
$joinRows[] = [
$parentIDField => $joinThroughObj->$parentIDField,
$childIDField => $joinThroughObj->$childIDField
];
$relationItemIDs[] = $joinThroughObj->$childField;
$relationItemIDs[] = $joinThroughObj->$childIDField;
}
// many_many + belongs_many_many
} else {
$joinTableQuery = DB::query('SELECT * FROM "' . $join . '" WHERE "' . $parentField . '" IN (' . implode(',', $ids) . ')');
$relationItemIDs = [];
$rows = [];
while ($row = $joinTableQuery->record()) {
$rows[] = [
$parentField => $row[$parentField],
$childField => $row[$childField]
];
$relationItemIDs[] = $row[$childField];
$joinTableQuery = DB::query('SELECT * FROM "' . $join . '" WHERE "' . $parentIDField . '" IN (' . implode(',', $parentIDs) . ')');
$relationItemIDs = $joinTableQuery->column($childIDField);
$joinRows = $joinTableQuery;
}

// Get ALL of the items for this relation up front, for ALL of the parents
// Fetched as a map so we can get the ID for all records up front (instead of in another nested loop)
// Fetched after that as an array because for some reason that performs better in the loop
$relationArray = DataObject::get($relationDataClass)->byIDs($relationItemIDs)->map('ID', 'Me')->toArray();

// Build a map of which children belong to which parent
$map = [];
foreach ($joinRows as $row) {
$parentID = $row[$parentIDField];
$childID = $row[$childIDField];
if (!isset($map[$parentID])) {
$map[$parentID] = [];
}
$map[$parentID][] = $relationArray[$childID];
}
$relationArray = DataObject::get($relationDataClass)->filter(['ID' => $relationItemIDs])->toArray();
foreach ($rows as $row) {
$eagerLoadID = $row[$parentField];
if (!isset($this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation])) {
$arrayList = ArrayList::create();
$arrayList->setDataClass($manyManyLastComponent['childClass']);
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation] = $arrayList;

// Store the children in an ArrayList against the correct parent
foreach ($map as $parentID => $children) {
// There shouldn't be a list for this relation already - if there is, something went wrong.
if (isset($this->eagerLoadedData[$eagerLoadRelation][$parentID][$relationName])) {
throw new LogicException("Relation list for $relationName on $parentDataClass cannot be fetched twice");
}
$relationItem = array_values(array_filter($relationArray, function ($relationItem) use ($row, $childField) {
return $relationItem->ID === $row[$childField];
}))[0];
$this->eagerLoadedData[$eagerLoadRelation][$eagerLoadID][$relation]->push($relationItem);

$arrayList = ArrayList::create($children);
$this->eagerLoadedData[$eagerLoadRelation][$parentID][$relationName] = $arrayList;
}

return [$relationArray, $relationItemIDs];
}

Expand Down Expand Up @@ -1305,15 +1324,16 @@ public function eagerLoad(...$relations): static
$message = "Eager loading only supports up to 3 levels of nesting, passed $count levels - $relation";
throw new InvalidArgumentException($message);
}
for ($i = 0; $i < $count; $i++) {
if ($i === 0) {
$arr[] = $parts[$i];
} else {
$arr[] = $arr[count($arr) - 1] . '.' . $parts[$i];
}
// Add each relation in the chain as its own entry to be eagerloaded
// e.g. for "Players.Teams.Coaches" you'll have three entries:
// "Players", "Players.Teams", and "Players.Teams.Coaches
$usedParts = [];
foreach ($parts as $part) {
$usedParts[] = $part;
$arr[] = implode('.', $usedParts);
}
}
$this->eagerLoadRelations = array_merge($this->eagerLoadRelations, $arr);
$this->eagerLoadRelations = array_unique(array_merge($this->eagerLoadRelations, $arr));
return $this;
}

Expand Down

0 comments on commit c29d7cf

Please sign in to comment.