Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(mongodb): Add pagination metadata to the aggregation results #6912

Merged
merged 4 commits into from
Jan 29, 2025
+133 −209
Merged

feat(mongodb): Add pagination metadata to the aggregation results #6912

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5ae3525
feat(mongodb): add pagination metadata to the aggregation results
GromNaN Dec 18, 2024
e131991
Fix PaginatorTest, CS and rename metadata fields for more unicity
GromNaN Jan 17, 2025
2da7970
Remove AddFields stubs
GromNaN Jan 17, 2025
b2c2212
Fix logic for $limit=0
GromNaN Jan 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion composer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@
"doctrine/common": "^3.2.2",
"doctrine/dbal": "^4.0",
"doctrine/doctrine-bundle": "^2.11",
"doctrine/mongodb-odm": "^2.6",
"doctrine/mongodb-odm": "^2.9.2",
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Required for mocking AddFields. See release 2.9.2

"doctrine/mongodb-odm-bundle": "^4.0 || ^5.0",
"doctrine/orm": "^2.17 || ^3.0",
"elasticsearch/elasticsearch": "^8.4",
Expand Down
30 changes: 16 additions & 14 deletions src/Doctrine/Odm/Extension/PaginationExtension.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,23 +63,25 @@ public function applyToCollection(Builder $aggregationBuilder, string $resourceC
* @var DocumentRepository
*/
$repository = $manager->getRepository($resourceClass);
$resultsAggregationBuilder = $repository->createAggregationBuilder()->skip($offset);

$facet = $aggregationBuilder->facet();
$addFields = $aggregationBuilder->addFields();

// Get the results slice, from $offset to $offset + $limit
// MongoDB does not support $limit: O, so we return an empty array directly
if ($limit > 0) {
$resultsAggregationBuilder->limit($limit);
$facet->field('results')->pipeline($repository->createAggregationBuilder()->skip($offset)->limit($limit));
} else {
// Results have to be 0 but MongoDB does not support a limit equal to 0.
$resultsAggregationBuilder->match()->field(Paginator::LIMIT_ZERO_MARKER_FIELD)->equals(Paginator::LIMIT_ZERO_MARKER);
$addFields->field('results')->literal([]);
}

$aggregationBuilder
->facet()
->field('results')->pipeline(
$resultsAggregationBuilder
)
->field('count')->pipeline(
$repository->createAggregationBuilder()
->count('count')
);
// Count the total number of items
$facet->field('count')->pipeline($repository->createAggregationBuilder()->count('count'));

// Store pagination metadata, read by the Paginator
// Using __ to avoid field names mapping
$addFields->field('__api_first_result__')->literal($offset);
$addFields->field('__api_max_results__')->literal($limit);
}

/**
Expand Down Expand Up @@ -109,7 +111,7 @@ public function getResult(Builder $aggregationBuilder, string $resourceClass, ?O
$attribute = $operation?->getExtraProperties()['doctrine_mongodb'] ?? [];
$executeOptions = $attribute['execute_options'] ?? [];

return new Paginator($aggregationBuilder->execute($executeOptions), $manager->getUnitOfWork(), $resourceClass, $aggregationBuilder->getPipeline());
return new Paginator($aggregationBuilder->execute($executeOptions), $manager->getUnitOfWork(), $resourceClass);
}

private function addCountToContext(Builder $aggregationBuilder, array $context): array
Expand Down
92 changes: 31 additions & 61 deletions src/Doctrine/Odm/Paginator.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

namespace ApiPlatform\Doctrine\Odm;

use ApiPlatform\Metadata\Exception\InvalidArgumentException;
use ApiPlatform\Metadata\Exception\RuntimeException;
use ApiPlatform\State\Pagination\HasNextPagePaginatorInterface;
use ApiPlatform\State\Pagination\PaginatorInterface;
use Doctrine\ODM\MongoDB\Iterator\Iterator;
Expand All @@ -27,29 +27,42 @@
*/
final class Paginator implements \IteratorAggregate, PaginatorInterface, HasNextPagePaginatorInterface
{
public const LIMIT_ZERO_MARKER_FIELD = '___';
public const LIMIT_ZERO_MARKER = 'limit0';
Comment on lines -30 to -31
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This constants are removed. They should have been marked as internal. We can keep them for backward compatibility, but they are useless now that they are not used in the aggregation pipeline.


private ?\ArrayIterator $iterator = null;
private readonly \ArrayIterator $iterator;

private readonly int $firstResult;

private readonly int $maxResults;

private readonly int $totalItems;

public function __construct(private readonly Iterator $mongoDbOdmIterator, private readonly UnitOfWork $unitOfWork, private readonly string $resourceClass, private readonly array $pipeline)
private readonly int $count;

public function __construct(Iterator $mongoDbOdmIterator, UnitOfWork $unitOfWork, string $resourceClass)
{
$resultsFacetInfo = $this->getFacetInfo('results');
$this->getFacetInfo('count');

/*
* Since the {@see \MongoDB\Driver\Cursor} class does not expose information about
* skip/limit parameters of the query, the values set in the facet stage are used instead.
*/
$this->firstResult = $this->getStageInfo($resultsFacetInfo, '$skip');
$this->maxResults = $this->hasLimitZeroStage($resultsFacetInfo) ? 0 : $this->getStageInfo($resultsFacetInfo, '$limit');
$this->totalItems = $mongoDbOdmIterator->toArray()[0]['count'][0]['count'] ?? 0;
$result = $mongoDbOdmIterator->toArray()[0];

if (array_diff_key(['results' => 1, 'count' => 1, '__api_first_result__' => 1, '__api_max_results__' => 1], $result)) {
throw new RuntimeException('The result of the query must contain only "__api_first_result__", "__api_max_results__", "results" and "count" fields.');
}

// The "count" facet contains the total number of documents,
// it is not set when the query does not return any document
$this->totalItems = $result['count'][0]['count'] ?? 0;

// The "results" facet contains the returned documents
if ([] === $result['results']) {
$this->count = 0;
$this->iterator = new \ArrayIterator();
} else {
$this->count = \count($result['results']);
$this->iterator = new \ArrayIterator(array_map(
static fn ($result): object => $unitOfWork->getOrCreateDocument($resourceClass, $result),
$result['results'],
));
}

$this->firstResult = $result['__api_first_result__'];
$this->maxResults = $result['__api_max_results__'];
}

/**
Expand Down Expand Up @@ -97,15 +110,15 @@ public function getTotalItems(): float
*/
public function getIterator(): \Traversable
{
return $this->iterator ?? $this->iterator = new \ArrayIterator(array_map(fn ($result): object => $this->unitOfWork->getOrCreateDocument($this->resourceClass, $result), $this->mongoDbOdmIterator->toArray()[0]['results']));
return $this->iterator;
}

/**
* {@inheritdoc}
*/
public function count(): int
{
return is_countable($this->mongoDbOdmIterator->toArray()[0]['results']) ? \count($this->mongoDbOdmIterator->toArray()[0]['results']) : 0;
return $this->count;
}

/**
Expand All @@ -115,47 +128,4 @@ public function hasNextPage(): bool
{
return $this->getLastPage() > $this->getCurrentPage();
}

/**
* @throws InvalidArgumentException
*/
private function getFacetInfo(string $field): array
{
foreach ($this->pipeline as $indexStage => $infoStage) {
if (\array_key_exists('$facet', $infoStage)) {
if (!isset($this->pipeline[$indexStage]['$facet'][$field])) {
throw new InvalidArgumentException("\"$field\" facet was not applied to the aggregation pipeline.");
}

return $this->pipeline[$indexStage]['$facet'][$field];
}
}

throw new InvalidArgumentException('$facet stage was not applied to the aggregation pipeline.');
}

/**
* @throws InvalidArgumentException
*/
private function getStageInfo(array $resultsFacetInfo, string $stage): int
{
foreach ($resultsFacetInfo as $resultFacetInfo) {
if (isset($resultFacetInfo[$stage])) {
return $resultFacetInfo[$stage];
}
}

throw new InvalidArgumentException("$stage stage was not applied to the facet stage of the aggregation pipeline.");
}

private function hasLimitZeroStage(array $resultsFacetInfo): bool
{
foreach ($resultsFacetInfo as $resultFacetInfo) {
if (self::LIMIT_ZERO_MARKER === ($resultFacetInfo['$match'][self::LIMIT_ZERO_MARKER_FIELD] ?? null)) {
return true;
}
}

return false;
}
}
75 changes: 51 additions & 24 deletions src/Doctrine/Odm/Tests/Extension/PaginationExtensionTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
namespace ApiPlatform\Doctrine\Odm\Tests\Extension;

use ApiPlatform\Doctrine\Odm\Extension\PaginationExtension;
use ApiPlatform\Doctrine\Odm\Paginator;
use ApiPlatform\Doctrine\Odm\Tests\DoctrineMongoDbOdmSetup;
use ApiPlatform\Doctrine\Odm\Tests\Fixtures\Document\Dummy;
use ApiPlatform\Metadata\Exception\InvalidArgumentException;
Expand All @@ -23,9 +22,9 @@
use ApiPlatform\State\Pagination\PaginatorInterface;
use ApiPlatform\State\Pagination\PartialPaginatorInterface;
use Doctrine\ODM\MongoDB\Aggregation\Builder;
use Doctrine\ODM\MongoDB\Aggregation\Stage\AddFields;
use Doctrine\ODM\MongoDB\Aggregation\Stage\Count;
use Doctrine\ODM\MongoDB\Aggregation\Stage\Facet;
use Doctrine\ODM\MongoDB\Aggregation\Stage\MatchStage as AggregationMatch;
use Doctrine\ODM\MongoDB\Aggregation\Stage\Skip;
use Doctrine\ODM\MongoDB\DocumentManager;
use Doctrine\ODM\MongoDB\Iterator\Iterator;
Expand All @@ -42,6 +41,7 @@ class PaginationExtensionTest extends TestCase
{
use ProphecyTrait;

/** @var ObjectProphecy<ManagerRegistry> */
private ObjectProphecy $managerRegistryProphecy;

/**
Expand Down Expand Up @@ -322,11 +322,14 @@ public function testGetResult(): void
$iteratorProphecy = $this->prophesize(Iterator::class);
$iteratorProphecy->toArray()->willReturn([
[
'results' => [],
'count' => [
[
'count' => 9,
],
],
'__api_first_result__' => 3,
'__api_max_results__' => 6,
],
]);

Expand All @@ -344,6 +347,12 @@ public function testGetResult(): void
],
],
],
[
'$addFields' => [
'__api_first_result__' => ['$literal' => 3],
'__api_max_results__' => ['$literal' => 6],
],
],
]);

$paginationExtension = new PaginationExtension(
Expand All @@ -370,11 +379,14 @@ public function testGetResultWithExecuteOptions(): void
$iteratorProphecy = $this->prophesize(Iterator::class);
$iteratorProphecy->toArray()->willReturn([
[
'results' => [],
'count' => [
[
'count' => 9,
],
],
'__api_first_result__' => 3,
'__api_max_results__' => 6,
],
]);

Expand All @@ -392,6 +404,12 @@ public function testGetResultWithExecuteOptions(): void
],
],
],
[
'$addFields' => [
'__api_first_result__' => ['$literal' => 3],
'__api_max_results__' => ['$literal' => 6],
],
],
]);

$paginationExtension = new PaginationExtension(
Expand All @@ -407,43 +425,52 @@ public function testGetResultWithExecuteOptions(): void

private function mockAggregationBuilder(int $expectedOffset, int $expectedLimit): ObjectProphecy
{
$skipProphecy = $this->prophesize(Skip::class);
if ($expectedLimit > 0) {
$skipProphecy->limit($expectedLimit)->shouldBeCalled();
} else {
$matchProphecy = $this->prophesize(AggregationMatch::class);
$matchProphecy->field(Paginator::LIMIT_ZERO_MARKER_FIELD)->shouldBeCalled()->willReturn($matchProphecy->reveal());
$matchProphecy->equals(Paginator::LIMIT_ZERO_MARKER)->shouldBeCalled()->willReturn($matchProphecy->reveal());
$skipProphecy->match()->shouldBeCalled()->willReturn($matchProphecy->reveal());
}

$resultsAggregationBuilderProphecy = $this->prophesize(Builder::class);
$resultsAggregationBuilderProphecy->skip($expectedOffset)->shouldBeCalled()->willReturn($skipProphecy->reveal());

$countProphecy = $this->prophesize(Count::class);

$countAggregationBuilderProphecy = $this->prophesize(Builder::class);
$countAggregationBuilderProphecy->count('count')->shouldBeCalled()->willReturn($countProphecy->reveal());

$repositoryProphecy = $this->prophesize(DocumentRepository::class);
$repositoryProphecy->createAggregationBuilder()->shouldBeCalled()->willReturn(
$resultsAggregationBuilderProphecy->reveal(),
$countAggregationBuilderProphecy->reveal()
);

$objectManagerProphecy = $this->prophesize(DocumentManager::class);
$objectManagerProphecy->getRepository('Foo')->shouldBeCalled()->willReturn($repositoryProphecy->reveal());

$this->managerRegistryProphecy->getManagerForClass('Foo')->shouldBeCalled()->willReturn($objectManagerProphecy->reveal());

$facetProphecy = $this->prophesize(Facet::class);
$facetProphecy->pipeline($skipProphecy)->shouldBeCalled()->willReturn($facetProphecy);
$facetProphecy->pipeline($countProphecy)->shouldBeCalled()->willReturn($facetProphecy);
$facetProphecy->field('count')->shouldBeCalled()->willReturn($facetProphecy);
$facetProphecy->field('results')->shouldBeCalled()->willReturn($facetProphecy);
$addFieldsProphecy = $this->prophesize(AddFields::class);

if ($expectedLimit > 0) {
$resultsAggregationBuilderProphecy = $this->prophesize(Builder::class);
$repositoryProphecy->createAggregationBuilder()->shouldBeCalled()->willReturn(
$resultsAggregationBuilderProphecy->reveal(),
$countAggregationBuilderProphecy->reveal()
);

$skipProphecy = $this->prophesize(Skip::class);
$skipProphecy->limit($expectedLimit)->shouldBeCalled()->willReturn($skipProphecy->reveal());
$resultsAggregationBuilderProphecy->skip($expectedOffset)->shouldBeCalled()->willReturn($skipProphecy->reveal());
$facetProphecy->field('results')->shouldBeCalled()->willReturn($facetProphecy);
$facetProphecy->pipeline($skipProphecy)->shouldBeCalled()->willReturn($facetProphecy->reveal());
} else {
$repositoryProphecy->createAggregationBuilder()->shouldBeCalled()->willReturn(
$countAggregationBuilderProphecy->reveal()
);

$addFieldsProphecy->field('results')->shouldBeCalled()->willReturn($addFieldsProphecy->reveal());
$addFieldsProphecy->literal([])->shouldBeCalled()->willReturn($addFieldsProphecy->reveal());
}

$facetProphecy->field('count')->shouldBeCalled()->willReturn($facetProphecy->reveal());
$facetProphecy->pipeline($countProphecy)->shouldBeCalled()->willReturn($facetProphecy->reveal());

$addFieldsProphecy->field('__api_first_result__')->shouldBeCalled()->willReturn($addFieldsProphecy->reveal());
$addFieldsProphecy->literal($expectedOffset)->shouldBeCalled()->willReturn($addFieldsProphecy->reveal());
$addFieldsProphecy->field('__api_max_results__')->shouldBeCalled()->willReturn($addFieldsProphecy->reveal());
$addFieldsProphecy->literal($expectedLimit)->shouldBeCalled()->willReturn($addFieldsProphecy->reveal());

$aggregationBuilderProphecy = $this->prophesize(Builder::class);
$aggregationBuilderProphecy->facet()->shouldBeCalled()->willReturn($facetProphecy->reveal());
$aggregationBuilderProphecy->addFields()->shouldBeCalled()->willReturn($addFieldsProphecy->reveal());

return $aggregationBuilderProphecy;
}
Expand Down
Loading
Loading