From 88966a35377215b9c49ba696e0bf6defa9429313 Mon Sep 17 00:00:00 2001 From: andriy Date: Sun, 30 Oct 2022 21:00:56 +0200 Subject: [PATCH 1/2] create with_fetch_all functional --- .../api_resources/abstract/resources.py | 59 +++++++++ .../abstract/tests/test_resources.py | 113 ++++++++++++++++++ crowdin_api/api_resources/bundles/resource.py | 4 +- .../api_resources/dictionaries/resource.py | 2 +- .../api_resources/distributions/resource.py | 2 +- .../api_resources/glossaries/resource.py | 6 +- crowdin_api/api_resources/groups/resource.py | 2 +- crowdin_api/api_resources/labels/resource.py | 2 +- .../api_resources/languages/resource.py | 2 +- .../machine_translation_engines/resource.py | 2 +- .../api_resources/projects/resource.py | 2 +- crowdin_api/api_resources/reports/resource.py | 2 +- .../api_resources/screenshots/resource.py | 4 +- .../api_resources/source_files/resource.py | 8 +- .../api_resources/source_strings/resource.py | 2 +- .../api_resources/storages/resource.py | 2 +- .../api_resources/string_comments/resource.py | 2 +- .../string_translations/resource.py | 8 +- crowdin_api/api_resources/tasks/resource.py | 4 +- crowdin_api/api_resources/teams/resource.py | 4 +- .../translation_memory/resource.py | 2 +- .../translation_status/resource.py | 12 +- .../api_resources/translations/resource.py | 2 +- crowdin_api/api_resources/users/resource.py | 2 +- crowdin_api/api_resources/vendors/resource.py | 2 +- .../api_resources/webhooks/resource.py | 2 +- .../api_resources/workflows/resource.py | 4 +- .../tests/test_workflows_resources.py | 2 +- crowdin_api/client.py | 49 ++++---- 29 files changed, 241 insertions(+), 68 deletions(-) diff --git a/crowdin_api/api_resources/abstract/resources.py b/crowdin_api/api_resources/abstract/resources.py index f2cd959..353e86c 100644 --- a/crowdin_api/api_resources/abstract/resources.py +++ b/crowdin_api/api_resources/abstract/resources.py @@ -8,6 +8,8 @@ class BaseResource(metaclass=ABCMeta): def __init__(self, requester: APIRequester, page_size=25): self.requester = requester self.page_size = page_size + self._flag_fetch_all = None + self._max_limit = None def _get_page_params(self, page: int): if page < 1: @@ -37,3 +39,60 @@ def get_page_params( raise ValueError("The limit must be greater than or equal to 1.") return {"offset": offset, "limit": limit} + + def with_fetch_all(self, max_limit: Optional[int] = None): + self._max_limit = max_limit + self._flag_fetch_all = True + return self + + def _get_entire_data(self, method: str, path: str, params: Optional[dict] = None): + if not self._flag_fetch_all: + return self.requester.request( + method=method, + path=path, + params=params, + ) + + contents = self._fetch_all( + method=method, + path=path, + params=params, + max_amount=self._max_limit + ) + self._flag_fetch_all = False + self._max_limit = None + return contents + + def _fetch_all( + self, + method: str, + path: str, + params: Optional[dict] = None, + max_amount: Optional[int] = None + ) -> list: + limit = 500 + offset = 0 + join_data = [] + if params is None: + params = {} + + if max_amount and max_amount < limit: + limit = max_amount + + while True: + params.update({"limit": limit, "offset": offset}) + + content = self.requester.request(method=method, path=path, params=params) + data = content.get("data", []) + data and join_data.extend(data) + + if len(data) < limit or (max_amount and len(join_data) >= max_amount): + break + else: + offset += limit + + if max_amount and max_amount < len(join_data) + limit: + limit = max_amount - len(join_data) + + content["data"] = join_data + return content diff --git a/crowdin_api/api_resources/abstract/tests/test_resources.py b/crowdin_api/api_resources/abstract/tests/test_resources.py index 5578572..e8b0d8e 100644 --- a/crowdin_api/api_resources/abstract/tests/test_resources.py +++ b/crowdin_api/api_resources/abstract/tests/test_resources.py @@ -1,3 +1,6 @@ +from unittest import mock +from unittest.mock import Mock + import pytest from crowdin_api.api_resources.abstract.resources import BaseResource from crowdin_api.requester import APIRequester @@ -31,3 +34,113 @@ def test_get_page_params_invalid_params(self, kwargs, base_absolut_url): resource = BaseResource(requester=APIRequester(base_url=base_absolut_url)) with pytest.raises(ValueError): resource.get_page_params(**kwargs) + + @pytest.mark.parametrize( + "in_param,out_param", + ( + ({"max_limit": None}, None), + ({"max_limit": 0}, 0), + ({"max_limit": 1}, 1), + ({"max_limit": 100}, 100), + ), + ) + def test_with_fetch_all(self, in_param, out_param, base_absolut_url): + resource = BaseResource(requester=APIRequester(base_url=base_absolut_url)) + + resource.with_fetch_all(**in_param) + + assert resource._max_limit == out_param + assert resource._flag_fetch_all is True + + @pytest.mark.parametrize( + "incoming_data, request_data", + ( + ( + {"method": "get", "path": ""}, + {"method": "get", "path": "", "params": None}, + ), + ( + {"method": "get", "path": "test", "params": "params"}, + {"method": "get", "path": "test", "params": "params"}, + ), + ), + ) + @mock.patch("crowdin_api.requester.APIRequester.request") + def test__get_list(self, m_request, incoming_data, request_data, base_absolut_url): + m_request.return_value = "response" + + resource = BaseResource(requester=APIRequester(base_url=base_absolut_url)) + + assert resource._get_entire_data(**incoming_data) == "response" + m_request.assert_called_once_with(**request_data) + + @pytest.mark.parametrize( + "max_limit, incoming_data, request_data", + ( + ( + None, + {"method": "get", "path": ""}, + {"method": "get", "path": "", "params": None, "max_amount": None}, + ), + ( + 0, + {"method": "get", "path": "test", "params": "params"}, + {"method": "get", "path": "test", "params": "params", "max_amount": 0}, + ), + ( + 1, + {"method": "get", "path": "test", "params": "params"}, + {"method": "get", "path": "test", "params": "params", "max_amount": 1}, + ), + ), + ) + def test__get_list_with__flag_fetch_all( + self, + incoming_data, + max_limit, + request_data, + base_absolut_url + ): + resource = BaseResource(requester=APIRequester(base_url=base_absolut_url)) + resource.with_fetch_all(max_limit=max_limit) + + resource._fetch_all = Mock(return_value="response") + + assert resource._flag_fetch_all is True + assert resource._get_entire_data(**incoming_data) == "response" + assert resource._flag_fetch_all is False + assert resource._max_limit is None + resource._fetch_all.assert_called_once_with(**request_data) + + @pytest.mark.parametrize( + "incoming_data, expected_result", + ( + ( + {"method": "get", "path": ""}, + {"data": [None] * 1} + ), + ( + {"method": "get", "path": ""}, + {"data": [None] * 499} + ), + ( + {"method": "get", "path": "", "params": None, "max_amount": 0}, + {"data": []} + ), + ( + {"method": "get", "path": "", "params": None, "max_amount": 1}, + {"data": [None] * 1} + ), + ( + {"method": "get", "path": "", "params": None, "max_amount": 499}, + {"data": [None] * 499} + ), + ), + ) + @mock.patch("crowdin_api.requester.APIRequester.request") + def test__fetch_all(self, m_request, incoming_data, expected_result, base_absolut_url): + m_request.return_value = expected_result + resource = BaseResource(requester=APIRequester(base_url=base_absolut_url)) + + testing_result = resource._fetch_all(**incoming_data) + assert testing_result == expected_result diff --git a/crowdin_api/api_resources/bundles/resource.py b/crowdin_api/api_resources/bundles/resource.py index d77ef47..e72b66b 100644 --- a/crowdin_api/api_resources/bundles/resource.py +++ b/crowdin_api/api_resources/bundles/resource.py @@ -39,7 +39,7 @@ def list_bundles( params = self.get_page_params(offset=offset, limit=limit) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_bundles_path(projectId=projectId), params=params, @@ -146,7 +146,7 @@ def get_bundle_list_files( params = self.get_page_params(offset=offset, limit=limit) - return self.requester.request( + return self._get_entire_data( method="get", path=f"{self.get_bundles_path(projectId=projectId, bundleId=bundleId)}/files", params=params, diff --git a/crowdin_api/api_resources/dictionaries/resource.py b/crowdin_api/api_resources/dictionaries/resource.py index 7923c31..497cee4 100644 --- a/crowdin_api/api_resources/dictionaries/resource.py +++ b/crowdin_api/api_resources/dictionaries/resource.py @@ -34,7 +34,7 @@ def list_dictionaries( params = self.get_page_params(page=page, offset=offset, limit=limit) params["languageIds"] = None if languageIds is None else ",".join(languageIds) - return self.requester.request( + return self._get_entire_data( method="get", path=f"projects/{projectId}/dictionaries", params=params, diff --git a/crowdin_api/api_resources/distributions/resource.py b/crowdin_api/api_resources/distributions/resource.py index 13e64f5..cd0931a 100644 --- a/crowdin_api/api_resources/distributions/resource.py +++ b/crowdin_api/api_resources/distributions/resource.py @@ -32,7 +32,7 @@ def list_distributions( https://developer.crowdin.com/api/v2/#operation/api.projects.distributions.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_distributions_path(projectId=projectId), params=self.get_page_params(offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/glossaries/resource.py b/crowdin_api/api_resources/glossaries/resource.py index 6080c92..71cda32 100644 --- a/crowdin_api/api_resources/glossaries/resource.py +++ b/crowdin_api/api_resources/glossaries/resource.py @@ -49,7 +49,7 @@ def list_glossaries( https://developer.crowdin.com/api/v2/#operation/api.glossaries.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_glossaries_path(), params=self.get_page_params(page=page, offset=offset, limit=limit), @@ -229,7 +229,7 @@ def list_terms( params.update(self.get_page_params(offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_terms_path(glossaryId=glossaryId), params=params, @@ -354,7 +354,7 @@ def list_concepts( https://developer.crowdin.com/api/v2/#operation/api.glossaries.concepts.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_concepts_path(glossaryId=glossaryId), params=self.get_page_params(offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/groups/resource.py b/crowdin_api/api_resources/groups/resource.py index abc5332..9986e50 100644 --- a/crowdin_api/api_resources/groups/resource.py +++ b/crowdin_api/api_resources/groups/resource.py @@ -77,7 +77,7 @@ def list_groups( params = {"parentId": parentId} params.update(self.get_page_params(offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_groups_path(), params=params, diff --git a/crowdin_api/api_resources/labels/resource.py b/crowdin_api/api_resources/labels/resource.py index abcaf61..87ce376 100644 --- a/crowdin_api/api_resources/labels/resource.py +++ b/crowdin_api/api_resources/labels/resource.py @@ -32,7 +32,7 @@ def list_labels( https://developer.crowdin.com/api/v2/#operation/api.projects.labels.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_labels_path(projectId=projectId), params=self.get_page_params(page=page, offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/languages/resource.py b/crowdin_api/api_resources/languages/resource.py index 3b8e478..bc24574 100644 --- a/crowdin_api/api_resources/languages/resource.py +++ b/crowdin_api/api_resources/languages/resource.py @@ -36,7 +36,7 @@ def list_supported_languages( https://developer.crowdin.com/api/v2/#operation/api.languages.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_languages_path(), params=self.get_page_params(page=page, offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/machine_translation_engines/resource.py b/crowdin_api/api_resources/machine_translation_engines/resource.py index d9cdd1f..3f52d34 100644 --- a/crowdin_api/api_resources/machine_translation_engines/resource.py +++ b/crowdin_api/api_resources/machine_translation_engines/resource.py @@ -32,7 +32,7 @@ def list_mts(self, limit: Optional[int] = None, offset: Optional[int] = None): https://developer.crowdin.com/api/v2/#operation/api.mts.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_mts_path(), params=self.get_page_params(offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/projects/resource.py b/crowdin_api/api_resources/projects/resource.py index fe171d7..8619fbd 100644 --- a/crowdin_api/api_resources/projects/resource.py +++ b/crowdin_api/api_resources/projects/resource.py @@ -51,7 +51,7 @@ def list_projects( params = {"userId": userId, "hasManagerAccess": hasManagerAccess} params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request(method="get", path=self.get_projects_path(), params=params) + return self._get_entire_data(method="get", path=self.get_projects_path(), params=params) def add_project(self, request_data: Dict): """ diff --git a/crowdin_api/api_resources/reports/resource.py b/crowdin_api/api_resources/reports/resource.py index 615e513..5fe5d2e 100644 --- a/crowdin_api/api_resources/reports/resource.py +++ b/crowdin_api/api_resources/reports/resource.py @@ -188,7 +188,7 @@ def list_report_settings_template( https://developer.crowdin.com/enterprise/api/v2/#operation/api.projects.reports.settings-templates.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_report_settings_templates_path(projectId=projectId), params=self.get_page_params(offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/screenshots/resource.py b/crowdin_api/api_resources/screenshots/resource.py index 1d523d8..8b5665e 100644 --- a/crowdin_api/api_resources/screenshots/resource.py +++ b/crowdin_api/api_resources/screenshots/resource.py @@ -42,7 +42,7 @@ def list_screenshots( https://developer.crowdin.com/api/v2/#operation/api.projects.screenshots.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_screenshots_path(projectId=projectId), params=self.get_page_params(page=page, offset=offset, limit=limit), @@ -153,7 +153,7 @@ def list_tags( https://developer.crowdin.com/api/v2/#operation/api.projects.screenshots.tags.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_tags_path(projectId=projectId, screenshotId=screenshotId), params=self.get_page_params(page=page, offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/source_files/resource.py b/crowdin_api/api_resources/source_files/resource.py index 15819f4..63eca54 100644 --- a/crowdin_api/api_resources/source_files/resource.py +++ b/crowdin_api/api_resources/source_files/resource.py @@ -56,7 +56,7 @@ def list_project_branches( params = {"name": name} params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_branch_path(projectId=projectId), params=params ) @@ -159,7 +159,7 @@ def list_directories( } params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_directory_path(projectId=projectId), params=params, @@ -270,7 +270,7 @@ def list_files( } params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_file_path(projectId=projectId), params=params ) @@ -441,7 +441,7 @@ def list_file_revisions( https://developer.crowdin.com/api/v2/#operation/api.projects.files.revisions.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_file_revisions_path(projectId=projectId, fileId=fileId), params=self.get_page_params(page=page, offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/source_strings/resource.py b/crowdin_api/api_resources/source_strings/resource.py index bbde80e..a30acf4 100644 --- a/crowdin_api/api_resources/source_strings/resource.py +++ b/crowdin_api/api_resources/source_strings/resource.py @@ -59,7 +59,7 @@ def list_strings( } params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_source_strings_path(projectId=projectId), params=params, diff --git a/crowdin_api/api_resources/storages/resource.py b/crowdin_api/api_resources/storages/resource.py index 8a61a03..363eec7 100644 --- a/crowdin_api/api_resources/storages/resource.py +++ b/crowdin_api/api_resources/storages/resource.py @@ -38,7 +38,7 @@ def list_storages( https://developer.crowdin.com/api/v2/#operation/api.storages.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_storages_path(), params=self.get_page_params(page=page, offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/string_comments/resource.py b/crowdin_api/api_resources/string_comments/resource.py index a6ac0b0..7e642a6 100644 --- a/crowdin_api/api_resources/string_comments/resource.py +++ b/crowdin_api/api_resources/string_comments/resource.py @@ -51,7 +51,7 @@ def list_string_comments( } params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_string_comments_path(projectId=projectId), params=params, diff --git a/crowdin_api/api_resources/string_translations/resource.py b/crowdin_api/api_resources/string_translations/resource.py index 38a5adc..0a1b356 100644 --- a/crowdin_api/api_resources/string_translations/resource.py +++ b/crowdin_api/api_resources/string_translations/resource.py @@ -48,7 +48,7 @@ def list_translation_approvals( } params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_approvals_path(projectId=projectId), params=params, @@ -132,7 +132,7 @@ def list_language_translations( } params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=f"projects/{projectId}/languages/{languageId}/translations", params=params, @@ -169,7 +169,7 @@ def list_string_translations( } params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_translations_path(projectId=projectId), params=params, @@ -285,7 +285,7 @@ def list_translation_votes( } params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_translation_votes_path(projectId=projectId), params=params, diff --git a/crowdin_api/api_resources/tasks/resource.py b/crowdin_api/api_resources/tasks/resource.py index edd6230..e7709e1 100644 --- a/crowdin_api/api_resources/tasks/resource.py +++ b/crowdin_api/api_resources/tasks/resource.py @@ -61,7 +61,7 @@ def list_tasks( params = {"assigneeId": assigneeId, "status": status} params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_tasks_path(projectId=projectId), params=params, @@ -330,7 +330,7 @@ def list_user_tasks( params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request(method="get", path="user/tasks", params=params) + return self._get_entire_data(method="get", path="user/tasks", params=params) def edit_task_archived_status(self, taskId: int, projectId: int, isArchived: bool = True): """ diff --git a/crowdin_api/api_resources/teams/resource.py b/crowdin_api/api_resources/teams/resource.py index 32eeaf3..26336c6 100644 --- a/crowdin_api/api_resources/teams/resource.py +++ b/crowdin_api/api_resources/teams/resource.py @@ -62,7 +62,7 @@ def list_teams(self, offset: Optional[int] = None, limit: Optional[int] = None): https://developer.crowdin.com/enterprise/api/v2/#operation/api.teams.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_teams_path(), params=self.get_page_params(offset=offset, limit=limit), @@ -129,7 +129,7 @@ def teams_member_list( https://developer.crowdin.com/enterprise/api/v2/#operation/api.teams.members.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_members_path(teamId=teamId), params=self.get_page_params(offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/translation_memory/resource.py b/crowdin_api/api_resources/translation_memory/resource.py index e51cc48..3f34992 100644 --- a/crowdin_api/api_resources/translation_memory/resource.py +++ b/crowdin_api/api_resources/translation_memory/resource.py @@ -39,7 +39,7 @@ def list_tms( https://developer.crowdin.com/api/v2/#operation/api.tms.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_tms_path(), params=self.get_page_params(page=page, offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/translation_status/resource.py b/crowdin_api/api_resources/translation_status/resource.py index 3f61b1a..96adbc4 100644 --- a/crowdin_api/api_resources/translation_status/resource.py +++ b/crowdin_api/api_resources/translation_status/resource.py @@ -32,7 +32,7 @@ def get_branch_progress( https://developer.crowdin.com/api/v2/#operation/api.projects.branches.languages.progress.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=f"projects/{projectId}/branches/{branchId}/languages/progress", params=self.get_page_params(page=page, offset=offset, limit=limit), @@ -53,7 +53,7 @@ def get_directory_progress( https://developer.crowdin.com/api/v2/#operation/api.projects.directories.languages.progress.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=f"projects/{projectId}/directories/{directoryId}/languages/progress", params=self.get_page_params(page=page, offset=offset, limit=limit), @@ -74,7 +74,7 @@ def get_file_progress( https://developer.crowdin.com/api/v2/#operation/api.projects.files.languages.progress.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=f"projects/{projectId}/files/{fileId}/languages/progress", params=self.get_page_params(page=page, offset=offset, limit=limit), @@ -95,7 +95,7 @@ def get_language_progress( https://developer.crowdin.com/api/v2/#operation/api.projects.languages.files.progress.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=f"projects/{projectId}/languages/{languageId}/progress", params=self.get_page_params(page=page, offset=offset, limit=limit), @@ -119,7 +119,7 @@ def get_project_progress( params = {"languageIds": None if languageIds is None else ",".join(languageIds)} params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=f"projects/{projectId}/languages/progress", params=params, @@ -149,7 +149,7 @@ def list_qa_check_issues( } params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=f"projects/{projectId}/languages/progress", params=params, diff --git a/crowdin_api/api_resources/translations/resource.py b/crowdin_api/api_resources/translations/resource.py index 8844bb1..8af8b9c 100644 --- a/crowdin_api/api_resources/translations/resource.py +++ b/crowdin_api/api_resources/translations/resource.py @@ -156,7 +156,7 @@ def list_project_builds( params = {"branchId": branchId} params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_builds_path(projectId=projectId), params=params, diff --git a/crowdin_api/api_resources/users/resource.py b/crowdin_api/api_resources/users/resource.py index 0412c13..835520f 100644 --- a/crowdin_api/api_resources/users/resource.py +++ b/crowdin_api/api_resources/users/resource.py @@ -33,7 +33,7 @@ def list_project_members( params = {"search": search, "role": role, "languageId": languageId} params.update(self.get_page_params(page=page, offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=f"projects/{projectId}/members", params=params, diff --git a/crowdin_api/api_resources/vendors/resource.py b/crowdin_api/api_resources/vendors/resource.py index 3b31e94..7e07cf3 100644 --- a/crowdin_api/api_resources/vendors/resource.py +++ b/crowdin_api/api_resources/vendors/resource.py @@ -25,7 +25,7 @@ def list_vendors(self, offset: Optional[int] = None, limit: Optional[int] = None https://developer.crowdin.com/enterprise/api/v2/#operation/api.teams.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path="vendors", params=self.get_page_params(offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/webhooks/resource.py b/crowdin_api/api_resources/webhooks/resource.py index 3d523d2..8ed86a4 100644 --- a/crowdin_api/api_resources/webhooks/resource.py +++ b/crowdin_api/api_resources/webhooks/resource.py @@ -58,7 +58,7 @@ def list_webhooks( https://developer.crowdin.com/api/v2/#tag/Webhooks """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_webhooks_path(projectId=projectId), params=self.get_page_params(page=page, offset=offset, limit=limit), diff --git a/crowdin_api/api_resources/workflows/resource.py b/crowdin_api/api_resources/workflows/resource.py index d3ec699..6f80b21 100644 --- a/crowdin_api/api_resources/workflows/resource.py +++ b/crowdin_api/api_resources/workflows/resource.py @@ -38,7 +38,7 @@ def list_workflow_steps(self, projectId: int): https://developer.crowdin.com/enterprise/api/v2/#operation/api.projects.workflow-steps.getMany """ - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_workflow_steps_path(projectId=projectId), ) @@ -72,7 +72,7 @@ def list_workflow_templates( params = {"groupId": groupId} params.update(self.get_page_params(offset=offset, limit=limit)) - return self.requester.request( + return self._get_entire_data( method="get", path=self.get_workflow_templates_path(), params=params, diff --git a/crowdin_api/api_resources/workflows/tests/test_workflows_resources.py b/crowdin_api/api_resources/workflows/tests/test_workflows_resources.py index 659d120..f27a3d0 100644 --- a/crowdin_api/api_resources/workflows/tests/test_workflows_resources.py +++ b/crowdin_api/api_resources/workflows/tests/test_workflows_resources.py @@ -43,7 +43,7 @@ def test_list_workflow_steps(self, m_request, base_absolut_url): resource = self.get_resource(base_absolut_url) assert resource.list_workflow_steps(projectId=1) == "response" m_request.assert_called_once_with( - method="get", path=resource.get_workflow_steps_path(projectId=1) + method="get", path=resource.get_workflow_steps_path(projectId=1), params=None ) @mock.patch("crowdin_api.requester.APIRequester.request") diff --git a/crowdin_api/client.py b/crowdin_api/client.py index 2ab927b..28615ed 100644 --- a/crowdin_api/client.py +++ b/crowdin_api/client.py @@ -22,30 +22,31 @@ class CrowdinClient: PAGE_SIZE = 25 HEADERS = {} - def __init__(self, - # TODO: replace this with union type expressions - # once we do not have to support <3.10 anymore - organization: Optional[str] =None, - token: Optional[str]=None, - base_url: Optional[str]=None, - user_agent: Optional[str]=None, - page_size: Optional[int]=None, - timeout: Optional[int]=None, - retry_delay: Union[int, float, None]=None, - max_retries: Optional[int]=None, - http_protocol: Optional[str]=None, - headers: Optional[dict]=None - ): - self.ORGANIZATION=organization or self.ORGANIZATION - self.TOKEN=token or self.TOKEN - self.BASE_URL=base_url or self.BASE_URL - self.USER_AGENT=user_agent or self.USER_AGENT - self.PAGE_SIZE=page_size or self.PAGE_SIZE - self.TIMEOUT=timeout or self.TIMEOUT - self.RETRY_DELAY=retry_delay or self.RETRY_DELAY - self.MAX_RETRIES=max_retries or self.MAX_RETRIES - self.HTTP_PROTOCOL=http_protocol or self.HTTP_PROTOCOL - self.HEADERS=headers or self.HEADERS + def __init__( + self, + # TODO: replace this with union type expressions + # once we do not have to support <3.10 anymore + organization: Optional[str] = None, + token: Optional[str] = None, + base_url: Optional[str] = None, + user_agent: Optional[str] = None, + page_size: Optional[int] = None, + timeout: Optional[int] = None, + retry_delay: Union[int, float, None] = None, + max_retries: Optional[int] = None, + http_protocol: Optional[str] = None, + headers: Optional[dict] = None + ): + self.ORGANIZATION = organization or self.ORGANIZATION + self.TOKEN = token or self.TOKEN + self.BASE_URL = base_url or self.BASE_URL + self.USER_AGENT = user_agent or self.USER_AGENT + self.PAGE_SIZE = page_size or self.PAGE_SIZE + self.TIMEOUT = timeout or self.TIMEOUT + self.RETRY_DELAY = retry_delay or self.RETRY_DELAY + self.MAX_RETRIES = max_retries or self.MAX_RETRIES + self.HTTP_PROTOCOL = http_protocol or self.HTTP_PROTOCOL + self.HEADERS = headers or self.HEADERS self._api_requestor = None if self.ORGANIZATION is None: From 1854f22fddfaa2604f7a61db304420db9d1938e4 Mon Sep 17 00:00:00 2001 From: andriy Date: Mon, 31 Oct 2022 10:49:23 +0200 Subject: [PATCH 2/2] update README.md file for with_fetch_all functional --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 99c6d34..1eb87d2 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Crowdin API is a full-featured RESTful API that helps you to integrate localizat ## Table of Contents * [Installation](#installation) * [Quick Start](#quick-start) +* [Fetch all records](#fetch-all-records) * [Seeking Assistance](#seeking-assistance) * [Contributing](#contributing) * [License](#license) @@ -98,6 +99,22 @@ my_file = client.source_files.add_file(__project_id__, storage['data']['id'], fi print(my_file) ``` +### Fetch all records + +It is possible to fetch all records from paginatable methods (where we have limit and offset in arguments). + +```python +from crowdin_api import CrowdinClient + +client = CrowdinClient(token='__token__') + +# get all projects +print(client.projects.with_fetch_all().list_projects()) + +# get projects but not more than 1000 +print(client.projects.with_fetch_all(1000).list_projects()) +``` + ## Seeking Assistance If you find any problems or would like to suggest a feature, please read the [How can I contribute](https://github.com/crowdin/crowdin-api-client-python/blob/main/CONTRIBUTING.md#how-can-i-contribute) section in our contributing guidelines.