From 029410d608a89f7c349dd4af8f05d046d3ce9047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Fri, 17 Jun 2022 11:41:36 -0500 Subject: [PATCH] docs: Add documentation for new implementation --- ...singer_sdk.pagination.BaseAPIPaginator.rst | 7 ++ ...er_sdk.pagination.BaseHATEOASPaginator.rst | 7 ++ ...ger_sdk.pagination.BaseOffsetPaginator.rst | 7 ++ ...sdk.pagination.BasePageNumberPaginator.rst | 7 ++ ...ger_sdk.pagination.HeaderLinkPaginator.rst | 7 ++ ...inger_sdk.pagination.JSONPathPaginator.rst | 7 ++ ...gination.LegacyPaginatedStreamProtocol.rst | 7 ++ ...r_sdk.pagination.LegacyStreamPaginator.rst | 7 ++ ...r_sdk.pagination.SimpleHeaderPaginator.rst | 7 ++ ...ger_sdk.pagination.SinglePagePaginator.rst | 7 ++ docs/porting.md | 68 ++++++++++++++++++- docs/reference.rst | 19 ++++++ 12 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 docs/classes/singer_sdk.pagination.BaseAPIPaginator.rst create mode 100644 docs/classes/singer_sdk.pagination.BaseHATEOASPaginator.rst create mode 100644 docs/classes/singer_sdk.pagination.BaseOffsetPaginator.rst create mode 100644 docs/classes/singer_sdk.pagination.BasePageNumberPaginator.rst create mode 100644 docs/classes/singer_sdk.pagination.HeaderLinkPaginator.rst create mode 100644 docs/classes/singer_sdk.pagination.JSONPathPaginator.rst create mode 100644 docs/classes/singer_sdk.pagination.LegacyPaginatedStreamProtocol.rst create mode 100644 docs/classes/singer_sdk.pagination.LegacyStreamPaginator.rst create mode 100644 docs/classes/singer_sdk.pagination.SimpleHeaderPaginator.rst create mode 100644 docs/classes/singer_sdk.pagination.SinglePagePaginator.rst diff --git a/docs/classes/singer_sdk.pagination.BaseAPIPaginator.rst b/docs/classes/singer_sdk.pagination.BaseAPIPaginator.rst new file mode 100644 index 000000000..f44e8d617 --- /dev/null +++ b/docs/classes/singer_sdk.pagination.BaseAPIPaginator.rst @@ -0,0 +1,7 @@ +singer_sdk.pagination.BaseAPIPaginator +====================================== + +.. currentmodule:: singer_sdk.pagination + +.. autoclass:: BaseAPIPaginator + :members: \ No newline at end of file diff --git a/docs/classes/singer_sdk.pagination.BaseHATEOASPaginator.rst b/docs/classes/singer_sdk.pagination.BaseHATEOASPaginator.rst new file mode 100644 index 000000000..b5a6f2b5e --- /dev/null +++ b/docs/classes/singer_sdk.pagination.BaseHATEOASPaginator.rst @@ -0,0 +1,7 @@ +singer_sdk.pagination.BaseHATEOASPaginator +========================================== + +.. currentmodule:: singer_sdk.pagination + +.. autoclass:: BaseHATEOASPaginator + :members: \ No newline at end of file diff --git a/docs/classes/singer_sdk.pagination.BaseOffsetPaginator.rst b/docs/classes/singer_sdk.pagination.BaseOffsetPaginator.rst new file mode 100644 index 000000000..ea7370d5b --- /dev/null +++ b/docs/classes/singer_sdk.pagination.BaseOffsetPaginator.rst @@ -0,0 +1,7 @@ +singer_sdk.pagination.BaseOffsetPaginator +========================================= + +.. currentmodule:: singer_sdk.pagination + +.. autoclass:: BaseOffsetPaginator + :members: \ No newline at end of file diff --git a/docs/classes/singer_sdk.pagination.BasePageNumberPaginator.rst b/docs/classes/singer_sdk.pagination.BasePageNumberPaginator.rst new file mode 100644 index 000000000..d33515ec3 --- /dev/null +++ b/docs/classes/singer_sdk.pagination.BasePageNumberPaginator.rst @@ -0,0 +1,7 @@ +singer_sdk.pagination.BasePageNumberPaginator +============================================= + +.. currentmodule:: singer_sdk.pagination + +.. autoclass:: BasePageNumberPaginator + :members: \ No newline at end of file diff --git a/docs/classes/singer_sdk.pagination.HeaderLinkPaginator.rst b/docs/classes/singer_sdk.pagination.HeaderLinkPaginator.rst new file mode 100644 index 000000000..f651a4116 --- /dev/null +++ b/docs/classes/singer_sdk.pagination.HeaderLinkPaginator.rst @@ -0,0 +1,7 @@ +singer_sdk.pagination.HeaderLinkPaginator +========================================= + +.. currentmodule:: singer_sdk.pagination + +.. autoclass:: HeaderLinkPaginator + :members: \ No newline at end of file diff --git a/docs/classes/singer_sdk.pagination.JSONPathPaginator.rst b/docs/classes/singer_sdk.pagination.JSONPathPaginator.rst new file mode 100644 index 000000000..59be0a5bf --- /dev/null +++ b/docs/classes/singer_sdk.pagination.JSONPathPaginator.rst @@ -0,0 +1,7 @@ +singer_sdk.pagination.JSONPathPaginator +======================================= + +.. currentmodule:: singer_sdk.pagination + +.. autoclass:: JSONPathPaginator + :members: \ No newline at end of file diff --git a/docs/classes/singer_sdk.pagination.LegacyPaginatedStreamProtocol.rst b/docs/classes/singer_sdk.pagination.LegacyPaginatedStreamProtocol.rst new file mode 100644 index 000000000..7a3f529de --- /dev/null +++ b/docs/classes/singer_sdk.pagination.LegacyPaginatedStreamProtocol.rst @@ -0,0 +1,7 @@ +singer_sdk.pagination.LegacyPaginatedStreamProtocol +=================================================== + +.. currentmodule:: singer_sdk.pagination + +.. autoclass:: LegacyPaginatedStreamProtocol + :members: \ No newline at end of file diff --git a/docs/classes/singer_sdk.pagination.LegacyStreamPaginator.rst b/docs/classes/singer_sdk.pagination.LegacyStreamPaginator.rst new file mode 100644 index 000000000..9d96831cd --- /dev/null +++ b/docs/classes/singer_sdk.pagination.LegacyStreamPaginator.rst @@ -0,0 +1,7 @@ +singer_sdk.pagination.LegacyStreamPaginator +=========================================== + +.. currentmodule:: singer_sdk.pagination + +.. autoclass:: LegacyStreamPaginator + :members: \ No newline at end of file diff --git a/docs/classes/singer_sdk.pagination.SimpleHeaderPaginator.rst b/docs/classes/singer_sdk.pagination.SimpleHeaderPaginator.rst new file mode 100644 index 000000000..4fce626e3 --- /dev/null +++ b/docs/classes/singer_sdk.pagination.SimpleHeaderPaginator.rst @@ -0,0 +1,7 @@ +singer_sdk.pagination.SimpleHeaderPaginator +=========================================== + +.. currentmodule:: singer_sdk.pagination + +.. autoclass:: SimpleHeaderPaginator + :members: \ No newline at end of file diff --git a/docs/classes/singer_sdk.pagination.SinglePagePaginator.rst b/docs/classes/singer_sdk.pagination.SinglePagePaginator.rst new file mode 100644 index 000000000..dc15a5d74 --- /dev/null +++ b/docs/classes/singer_sdk.pagination.SinglePagePaginator.rst @@ -0,0 +1,7 @@ +singer_sdk.pagination.SinglePagePaginator +========================================= + +.. currentmodule:: singer_sdk.pagination + +.. autoclass:: SinglePagePaginator + :members: \ No newline at end of file diff --git a/docs/porting.md b/docs/porting.md index 91e27796f..1c884435b 100644 --- a/docs/porting.md +++ b/docs/porting.md @@ -103,10 +103,76 @@ _Important: If you've gotten this far, this is a good time to commit your code b Pagination is generally unique for almost every API. There's no single method that solves for very different API's approach to pagination. -Most likely you will use `get_next_page_token` to parse and return whatever the "next page" token is for your source, and you'll use `get_url_params` to define how to pass the "next page" token back to the API when asking for subsequent pages. +Most likely you will use [get_new_paginator](singer_sdk.RESTStream.get_new_paginator) to instantiate a [pagination class](./classes/singer_sdk.pagination.BaseAPIPaginator) for your source, and you'll use `get_url_params` to define how to pass the "next page" token back to the API when asking for subsequent pages. When you think you have it right, run `poetry run tap-mysource` again, and debug until you are confident the result is including multiple pages back from the API. +You can also add unit tests for your pagination implementation for additional confidence: + +```python +from singer_sdk.pagination import BaseHATEOASPaginator, first + + +class CustomHATEOASPaginator(BaseHATEOASPaginator): + def get_next_url(self, response: Response) -> str | None: + """Get a parsed HATEOAS link for the next, if the response has one.""" + + try: + return first( + extract_jsonpath("$.links[?(@.rel=='next')].href", response.json()) + ) + except StopIteration: + return None + + +def test_paginator_custom_hateoas(): + """Validate paginator that my custom paginator.""" + + resource_path = "/path/to/resource" + response = Response() + paginator = CustomHATEOASPaginator() + assert not paginator.finished + assert paginator.current_value is None + assert paginator.count == 0 + + response._content = json.dumps( + { + "links": [ + { + "rel": "next", + "href": f"{resource_path}?page=2&limit=100", + } + ] + } + ).encode() + paginator.advance(response) + assert not paginator.finished + assert paginator.current_value.path == resource_path + assert paginator.current_value.query == "page=2&limit=100" + assert paginator.count == 1 + + response._content = json.dumps( + { + "links": [ + { + "rel": "next", + "href": f"{resource_path}?page=3&limit=100", + } + ] + } + ).encode() + paginator.advance(response) + assert not paginator.finished + assert paginator.current_value.path == resource_path + assert paginator.current_value.query == "page=3&limit=100" + assert paginator.count == 2 + + response._content = json.dumps({"links": []}).encode() + paginator.advance(response) + assert paginator.finished + assert paginator.count == 3 +``` + Note: Depending on how well the API is designed, this could take 5 minutes or multiple hours. If you need help, sometimes [PostMan](https://postman.com) or [Thunder Client](https://marketplace.visualstudio.com/items?itemName=rangav.vscode-thunder-client) can be helpful in debugging the APIs specific quirks. ## Run pytest diff --git a/docs/reference.rst b/docs/reference.rst index 1641d357c..09e7ec51b 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -88,3 +88,22 @@ JSON Schema builder classes :template: module.rst typing + + +Pagination +---------- + +.. autosummary:: + :toctree: classes + :template: class.rst + + pagination.BaseAPIPaginator + pagination.SinglePagePaginator + pagination.BaseHATEOASPaginator + pagination.HeaderLinkPaginator + pagination.JSONPathPaginator + pagination.SimpleHeaderPaginator + pagination.BasePageNumberPaginator + pagination.BaseOffsetPaginator + pagination.LegacyPaginatedStreamProtocol + pagination.LegacyStreamPaginator