Merge pull request #55 from crowdin/create_with_fetch_all

create with_fetch_all functional

README.md

@@ -25,6 +25,7 @@ Crowdin API is a full-featured RESTful API that helps you to integrate localizat
 ## Table of Contents
 * [Installation](#installation)
 * [Quick Start](#quick-start)
+* [Fetch all records](#fetch-all-records)
 * [Seeking Assistance](#seeking-assistance)
 * [Contributing](#contributing)
 * [License](#license)
@@ -98,6 +99,22 @@ my_file = client.source_files.add_file(__project_id__, storage['data']['id'], fi
 print(my_file)
 ```
 
+### Fetch all records
+
+It is possible to fetch all records from paginatable methods (where we have limit and offset in arguments).
+
+```python
+from crowdin_api import CrowdinClient
+
+client = CrowdinClient(token='__token__')
+
+# get all projects
+print(client.projects.with_fetch_all().list_projects())
+
+# get projects but not more than 1000
+print(client.projects.with_fetch_all(1000).list_projects())
+```
+
 ## Seeking Assistance
 
 If you find any problems or would like to suggest a feature, please read the [How can I contribute](https://github.com/crowdin/crowdin-api-client-python/blob/main/CONTRIBUTING.md#how-can-i-contribute) section in our contributing guidelines.

crowdin_api/api_resources/abstract/resources.py

@@ -8,6 +8,8 @@ class BaseResource(metaclass=ABCMeta):
     def __init__(self, requester: APIRequester, page_size=25):
         self.requester = requester
         self.page_size = page_size
+        self._flag_fetch_all = None
+        self._max_limit = None
 
     def _get_page_params(self, page: int):
         if page < 1:
@@ -37,3 +39,60 @@ def get_page_params(
                 raise ValueError("The limit must be greater than or equal to 1.")
 
         return {"offset": offset, "limit": limit}
+
+    def with_fetch_all(self, max_limit: Optional[int] = None):
+        self._max_limit = max_limit
+        self._flag_fetch_all = True
+        return self
+
+    def _get_entire_data(self, method: str, path: str, params: Optional[dict] = None):
+        if not self._flag_fetch_all:
+            return self.requester.request(
+                method=method,
+                path=path,
+                params=params,
+            )
+
+        contents = self._fetch_all(
+            method=method,
+            path=path,
+            params=params,
+            max_amount=self._max_limit
+        )
+        self._flag_fetch_all = False
+        self._max_limit = None
+        return contents
+
+    def _fetch_all(
+        self,
+        method: str,
+        path: str,
+        params: Optional[dict] = None,
+        max_amount: Optional[int] = None
+    ) -> list:
+        limit = 500
+        offset = 0
+        join_data = []
+        if params is None:
+            params = {}
+
+        if max_amount and max_amount < limit:
+            limit = max_amount
+
+        while True:
+            params.update({"limit": limit, "offset": offset})
+
+            content = self.requester.request(method=method, path=path, params=params)
+            data = content.get("data", [])
+            data and join_data.extend(data)
+
+            if len(data) < limit or (max_amount and len(join_data) >= max_amount):
+                break
+            else:
+                offset += limit
+
+            if max_amount and max_amount < len(join_data) + limit:
+                limit = max_amount - len(join_data)
+
+        content["data"] = join_data
+        return content

crowdin_api/api_resources/abstract/tests/test_resources.py

@@ -1,3 +1,6 @@
+from unittest import mock
+from unittest.mock import Mock
+
 import pytest
 from crowdin_api.api_resources.abstract.resources import BaseResource
 from crowdin_api.requester import APIRequester
@@ -31,3 +34,113 @@ def test_get_page_params_invalid_params(self, kwargs, base_absolut_url):
         resource = BaseResource(requester=APIRequester(base_url=base_absolut_url))
         with pytest.raises(ValueError):
             resource.get_page_params(**kwargs)
+
+    @pytest.mark.parametrize(
+        "in_param,out_param",
+        (
+            ({"max_limit": None}, None),
+            ({"max_limit": 0}, 0),
+            ({"max_limit": 1}, 1),
+            ({"max_limit": 100}, 100),
+        ),
+    )
+    def test_with_fetch_all(self, in_param, out_param, base_absolut_url):
+        resource = BaseResource(requester=APIRequester(base_url=base_absolut_url))
+
+        resource.with_fetch_all(**in_param)
+
+        assert resource._max_limit == out_param
+        assert resource._flag_fetch_all is True
+
+    @pytest.mark.parametrize(
+        "incoming_data, request_data",
+        (
+            (
+                {"method": "get", "path": ""},
+                {"method": "get", "path": "", "params": None},
+            ),
+            (
+                {"method": "get", "path": "test", "params": "params"},
+                {"method": "get", "path": "test", "params": "params"},
+            ),
+        ),
+    )
+    @mock.patch("crowdin_api.requester.APIRequester.request")
+    def test__get_list(self, m_request, incoming_data, request_data, base_absolut_url):
+        m_request.return_value = "response"
+
+        resource = BaseResource(requester=APIRequester(base_url=base_absolut_url))
+
+        assert resource._get_entire_data(**incoming_data) == "response"
+        m_request.assert_called_once_with(**request_data)
+
+    @pytest.mark.parametrize(
+        "max_limit, incoming_data, request_data",
+        (
+            (
+                None,
+                {"method": "get", "path": ""},
+                {"method": "get", "path": "", "params": None, "max_amount": None},
+            ),
+            (
+                0,
+                {"method": "get", "path": "test", "params": "params"},
+                {"method": "get", "path": "test", "params": "params", "max_amount": 0},
+            ),
+            (
+                1,
+                {"method": "get", "path": "test", "params": "params"},
+                {"method": "get", "path": "test", "params": "params", "max_amount": 1},
+            ),
+        ),
+    )
+    def test__get_list_with__flag_fetch_all(
+        self,
+        incoming_data,
+        max_limit,
+        request_data,
+        base_absolut_url
+    ):
+        resource = BaseResource(requester=APIRequester(base_url=base_absolut_url))
+        resource.with_fetch_all(max_limit=max_limit)
+
+        resource._fetch_all = Mock(return_value="response")
+
+        assert resource._flag_fetch_all is True
+        assert resource._get_entire_data(**incoming_data) == "response"
+        assert resource._flag_fetch_all is False
+        assert resource._max_limit is None
+        resource._fetch_all.assert_called_once_with(**request_data)
+
+    @pytest.mark.parametrize(
+        "incoming_data, expected_result",
+        (
+            (
+                {"method": "get", "path": ""},
+                {"data": [None] * 1}
+            ),
+            (
+                {"method": "get", "path": ""},
+                {"data": [None] * 499}
+            ),
+            (
+                {"method": "get", "path": "", "params": None, "max_amount": 0},
+                {"data": []}
+            ),
+            (
+                {"method": "get", "path": "", "params": None, "max_amount": 1},
+                {"data": [None] * 1}
+            ),
+            (
+                {"method": "get", "path": "", "params": None, "max_amount": 499},
+                {"data": [None] * 499}
+            ),
+        ),
+    )
+    @mock.patch("crowdin_api.requester.APIRequester.request")
+    def test__fetch_all(self, m_request, incoming_data, expected_result, base_absolut_url):
+        m_request.return_value = expected_result
+        resource = BaseResource(requester=APIRequester(base_url=base_absolut_url))
+
+        testing_result = resource._fetch_all(**incoming_data)
+        assert testing_result == expected_result

crowdin_api/api_resources/bundles/resource.py

@@ -39,7 +39,7 @@ def list_bundles(
 
         params = self.get_page_params(offset=offset, limit=limit)
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_bundles_path(projectId=projectId),
             params=params,
@@ -146,7 +146,7 @@ def get_bundle_list_files(
 
         params = self.get_page_params(offset=offset, limit=limit)
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=f"{self.get_bundles_path(projectId=projectId, bundleId=bundleId)}/files",
             params=params,

crowdin_api/api_resources/dictionaries/resource.py

@@ -34,7 +34,7 @@ def list_dictionaries(
         params = self.get_page_params(page=page, offset=offset, limit=limit)
         params["languageIds"] = None if languageIds is None else ",".join(languageIds)
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=f"projects/{projectId}/dictionaries",
             params=params,

crowdin_api/api_resources/distributions/resource.py

@@ -32,7 +32,7 @@ def list_distributions(
         https://developer.crowdin.com/api/v2/#operation/api.projects.distributions.getMany
         """
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_distributions_path(projectId=projectId),
             params=self.get_page_params(offset=offset, limit=limit),

crowdin_api/api_resources/glossaries/resource.py

@@ -49,7 +49,7 @@ def list_glossaries(
         https://developer.crowdin.com/api/v2/#operation/api.glossaries.getMany
         """
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_glossaries_path(),
             params=self.get_page_params(page=page, offset=offset, limit=limit),
@@ -229,7 +229,7 @@ def list_terms(
 
         params.update(self.get_page_params(offset=offset, limit=limit))
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_terms_path(glossaryId=glossaryId),
             params=params,
@@ -354,7 +354,7 @@ def list_concepts(
         https://developer.crowdin.com/api/v2/#operation/api.glossaries.concepts.getMany
         """
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_concepts_path(glossaryId=glossaryId),
             params=self.get_page_params(offset=offset, limit=limit),

crowdin_api/api_resources/groups/resource.py

@@ -77,7 +77,7 @@ def list_groups(
         params = {"parentId": parentId}
         params.update(self.get_page_params(offset=offset, limit=limit))
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_groups_path(),
             params=params,

crowdin_api/api_resources/labels/resource.py

@@ -32,7 +32,7 @@ def list_labels(
         https://developer.crowdin.com/api/v2/#operation/api.projects.labels.getMany
         """
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_labels_path(projectId=projectId),
             params=self.get_page_params(page=page, offset=offset, limit=limit),

crowdin_api/api_resources/languages/resource.py

@@ -36,7 +36,7 @@ def list_supported_languages(
         https://developer.crowdin.com/api/v2/#operation/api.languages.getMany
         """
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_languages_path(),
             params=self.get_page_params(page=page, offset=offset, limit=limit),

crowdin_api/api_resources/machine_translation_engines/resource.py

@@ -32,7 +32,7 @@ def list_mts(self, limit: Optional[int] = None, offset: Optional[int] = None):
         https://developer.crowdin.com/api/v2/#operation/api.mts.getMany
         """
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_mts_path(),
             params=self.get_page_params(offset=offset, limit=limit),

crowdin_api/api_resources/projects/resource.py

@@ -51,7 +51,7 @@ def list_projects(
         params = {"userId": userId, "hasManagerAccess": hasManagerAccess}
         params.update(self.get_page_params(page=page, offset=offset, limit=limit))
 
-        return self.requester.request(method="get", path=self.get_projects_path(), params=params)
+        return self._get_entire_data(method="get", path=self.get_projects_path(), params=params)
 
     def add_project(self, request_data: Dict):
         """

crowdin_api/api_resources/reports/resource.py

@@ -188,7 +188,7 @@ def list_report_settings_template(
         https://developer.crowdin.com/enterprise/api/v2/#operation/api.projects.reports.settings-templates.getMany
         """
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_report_settings_templates_path(projectId=projectId),
             params=self.get_page_params(offset=offset, limit=limit),

crowdin_api/api_resources/screenshots/resource.py

@@ -42,7 +42,7 @@ def list_screenshots(
         https://developer.crowdin.com/api/v2/#operation/api.projects.screenshots.getMany
         """
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_screenshots_path(projectId=projectId),
             params=self.get_page_params(page=page, offset=offset, limit=limit),
@@ -153,7 +153,7 @@ def list_tags(
         https://developer.crowdin.com/api/v2/#operation/api.projects.screenshots.tags.getMany
         """
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_tags_path(projectId=projectId, screenshotId=screenshotId),
             params=self.get_page_params(page=page, offset=offset, limit=limit),

crowdin_api/api_resources/source_files/resource.py

@@ -56,7 +56,7 @@ def list_project_branches(
         params = {"name": name}
         params.update(self.get_page_params(page=page, offset=offset, limit=limit))
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get", path=self.get_branch_path(projectId=projectId), params=params
         )
 
@@ -159,7 +159,7 @@ def list_directories(
         }
         params.update(self.get_page_params(page=page, offset=offset, limit=limit))
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_directory_path(projectId=projectId),
             params=params,
@@ -270,7 +270,7 @@ def list_files(
         }
         params.update(self.get_page_params(page=page, offset=offset, limit=limit))
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get", path=self.get_file_path(projectId=projectId), params=params
         )
 
@@ -441,7 +441,7 @@ def list_file_revisions(
         https://developer.crowdin.com/api/v2/#operation/api.projects.files.revisions.getMany
         """
 
-        return self.requester.request(
+        return self._get_entire_data(
             method="get",
             path=self.get_file_revisions_path(projectId=projectId, fileId=fileId),
             params=self.get_page_params(page=page, offset=offset, limit=limit),