From 76140b517d6311455b6438e1c4f476b4bab1d98a Mon Sep 17 00:00:00 2001 From: Yevhenii <34103125+yevhenii-ldv@users.noreply.github.com> Date: Fri, 17 Sep 2021 15:47:37 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Recharge:=20change=20cu?= =?UTF-8?q?rsor=5Ffield=20from=20created=5Fat=20on=20updated=5Fat=20(#6149?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Source Recharge: change cursor_field from created_at on updated_at --- .../45d2e135-2ede-49e1-939f-3e3ec357a65e.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../connectors/source-recharge/Dockerfile | 2 +- .../acceptance-test-config.yml | 5 ++-- .../integration_tests/abnormal_state.json | 14 +++++----- .../integration_tests/configured_catalog.json | 28 +++++++++---------- .../streams_with_output_records_catalog.json | 24 ++++++++-------- .../source-recharge/requirements.txt | 2 ++ .../sample_files/sample_state.json | 14 +++++----- .../source-recharge/source_recharge/api.py | 2 +- docs/integrations/sources/recharge.md | 6 ++++ 11 files changed, 55 insertions(+), 46 deletions(-) create mode 100644 airbyte-integrations/connectors/source-recharge/requirements.txt diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json index fba118d7b66a5..ba3528cd02e0f 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "45d2e135-2ede-49e1-939f-3e3ec357a65e", "name": "Recharge", "dockerRepository": "airbyte/source-recharge", - "dockerImageTag": "0.1.1", + "dockerImageTag": "0.1.2", "documentationUrl": "https://docs.airbyte.io/integrations/sources/recharge" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 2279ca821191a..f07efb5c70a08 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -324,7 +324,7 @@ - sourceDefinitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e name: Recharge dockerRepository: airbyte/source-recharge - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/recharge - sourceDefinitionId: fe2b4084-3386-4d3b-9ad6-308f61a6f1e6 name: Harvest diff --git a/airbyte-integrations/connectors/source-recharge/Dockerfile b/airbyte-integrations/connectors/source-recharge/Dockerfile index 43eafe69c3f2c..2c4b16295b8ea 100644 --- a/airbyte-integrations/connectors/source-recharge/Dockerfile +++ b/airbyte-integrations/connectors/source-recharge/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-recharge diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml index b9e9e49552c9c..8595868f33e24 100644 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml @@ -12,12 +12,13 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json" + timeout_seconds: 1200 incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json" future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - addresses: [ "created_at" ] + timeout_seconds: 900 full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 1200 diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json index fbedef9735c1d..ae9cea634a97e 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json @@ -1,23 +1,23 @@ { "addresses": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "charges": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "customers": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "discounts": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "onetimes": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "orders": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "subscriptions": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" } } diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json index f37ccc314ea27..e878274c1f9ba 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json @@ -6,12 +6,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -19,12 +19,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -42,12 +42,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -55,12 +55,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -78,12 +78,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -91,12 +91,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -124,12 +124,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] } ] } diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/streams_with_output_records_catalog.json b/airbyte-integrations/connectors/source-recharge/integration_tests/streams_with_output_records_catalog.json index abd316e473872..cf16288f6364c 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/streams_with_output_records_catalog.json +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/streams_with_output_records_catalog.json @@ -6,12 +6,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -19,12 +19,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -42,12 +42,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -55,12 +55,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -68,12 +68,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -101,12 +101,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] } ] } diff --git a/airbyte-integrations/connectors/source-recharge/requirements.txt b/airbyte-integrations/connectors/source-recharge/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-recharge/sample_files/sample_state.json b/airbyte-integrations/connectors/source-recharge/sample_files/sample_state.json index 47f6bb771a5e2..f667a07e67b93 100644 --- a/airbyte-integrations/connectors/source-recharge/sample_files/sample_state.json +++ b/airbyte-integrations/connectors/source-recharge/sample_files/sample_state.json @@ -1,23 +1,23 @@ { "addresses": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "charges": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "customers": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "discounts": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "onetimes": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "orders": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "subscriptions": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" } } diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index d4c2222d4ae93..e7a200eba0b5a 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -79,7 +79,7 @@ def get_stream_data(self, response_data: Any) -> List[dict]: class IncrementalRechargeStream(RechargeStream, ABC): - cursor_field = "created_at" + cursor_field = "updated_at" def __init__(self, start_date, **kwargs): super().__init__(**kwargs) diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md index f61fc14c9e6a7..fead348af5fd2 100644 --- a/docs/integrations/sources/recharge.md +++ b/docs/integrations/sources/recharge.md @@ -46,3 +46,9 @@ The Recharge connector should gracefully handle Recharge API limitations under n Please read [How to generate your API token](https://support.rechargepayments.com/hc/en-us/articles/360008829993-ReCharge-API-). +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :-------- | :----- | :------ | +| 0.1.2 | 2021-09-17 | [6149](https://github.com/airbytehq/airbyte/pull/6149) | Change `cursor_field` for Incremental streams | +| \ No newline at end of file