diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json index 24bf9de88414e..94ad93deff618 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9", "name": "Shopify", "dockerRepository": "airbyte/source-shopify", - "dockerImageTag": "0.1.16", + "dockerImageTag": "0.1.17", "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 1c88d1560a040..509359fb228c6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -160,7 +160,7 @@ - sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 name: Shopify dockerRepository: airbyte/source-shopify - dockerImageTag: 0.1.16 + dockerImageTag: 0.1.17 documentationUrl: https://docs.airbyte.io/integrations/sources/shopify - sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6 name: Redshift diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile index 2e1f154b1d550..fcd00bed3fd97 100644 --- a/airbyte-integrations/connectors/source-shopify/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify/Dockerfile @@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.17 LABEL io.airbyte.name=airbyte/source-shopify diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml index ab0170e8b4345..a8b2ef8d61a2c 100644 --- a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml @@ -13,7 +13,7 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" timeout_seconds: 1200 - incremental: + incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json index 7cb02047ef54d..3428f978b90bc 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json @@ -117,7 +117,7 @@ "properties": { "price_set": {}, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "title": { "type": ["null", "string"] @@ -143,7 +143,7 @@ "type": ["null", "array"] }, "total_line_items_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "closed_at": { "type": ["null", "string"], @@ -165,10 +165,10 @@ "type": ["null", "string"] }, "total_tax": { - "type": ["null", "string"] + "type": ["null", "number"] }, "subtotal_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "line_items": { "items": { @@ -181,7 +181,7 @@ "type": ["null", "integer"] }, "compare_at_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "destination_location_id": { "type": ["null", "integer"] @@ -190,7 +190,7 @@ "type": ["null", "string"] }, "line_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "origin_location_id": { "type": ["null", "integer"] @@ -237,7 +237,7 @@ }, "amount_set": {}, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -283,7 +283,7 @@ "properties": { "price_set": {}, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "title": { "type": ["null", "string"] @@ -338,7 +338,7 @@ "type": ["null", "object"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "requires_shipping": { "type": ["null", "boolean"] @@ -399,7 +399,7 @@ "type": ["null", "string"] }, "total_discounts": { - "type": ["null", "string"] + "type": ["null", "number"] }, "note": { "type": ["null", "string"] @@ -430,7 +430,7 @@ "type": ["null", "integer"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "requested_fulfillment_service_id": { "type": ["null", "string"] @@ -446,7 +446,7 @@ "properties": { "price_set": {}, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "title": { "type": ["null", "string"] @@ -630,7 +630,7 @@ "type": ["null", "boolean"] }, "total_spent": { - "type": ["null", "string"] + "type": ["null", "number"] }, "last_order_id": { "type": ["null", "integer"] @@ -746,7 +746,7 @@ } }, "total_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "cart_token": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json index 648b89423bb15..4a6c9a1b1219b 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json @@ -79,7 +79,7 @@ "type": ["null", "boolean"] }, "total_spent": { - "type": ["null", "string"] + "type": ["null", "number"] }, "last_order_id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json index 3b2c670c487cd..888b8a84e9847 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json @@ -42,7 +42,7 @@ "type": ["null", "string"] }, "total_spent": { - "type": ["null", "string"] + "type": ["null", "number"] }, "last_order_id": { "type": ["null", "integer"] @@ -359,7 +359,7 @@ } }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -368,7 +368,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -379,7 +379,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -413,7 +413,7 @@ "type": ["null", "string"] }, "total_discount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_discount_set": { "type": ["null", "object"], @@ -422,7 +422,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -433,7 +433,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -460,7 +460,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -469,7 +469,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -480,7 +480,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -515,7 +515,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "rate": { "type": ["null", "number"] @@ -530,7 +530,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -541,7 +541,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -559,7 +559,7 @@ "discounted_price_set": {}, "price_set": {}, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "title": { "type": ["null", "string"] @@ -614,7 +614,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "rate": { "type": ["null", "number"] @@ -629,7 +629,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -640,7 +640,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -662,7 +662,7 @@ "type": ["null", "boolean"] }, "total_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "completed_at": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json index ab5f611279d50..e852ef31d321b 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json @@ -70,7 +70,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -81,7 +81,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -97,7 +97,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -108,7 +108,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -134,7 +134,7 @@ "type": ["null", "boolean"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "tax_lines": { "type": ["null", "array"], @@ -148,7 +148,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -159,7 +159,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -168,7 +168,7 @@ "type": ["null", "object"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "title": { "type": ["null", "string"] @@ -226,7 +226,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -237,7 +237,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -249,7 +249,7 @@ "type": ["null", "string"] }, "pre_tax_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "variant_title": { "type": ["null", "string"] @@ -262,7 +262,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -273,7 +273,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -286,7 +286,7 @@ "items": { "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "amount_set": { "properties": { @@ -296,7 +296,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -307,7 +307,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -330,7 +330,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -341,7 +341,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -359,7 +359,7 @@ "type": ["null", "string"] }, "total_discount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "name": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json index 61cfde09f4649..fae5bb5acf8f0 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json @@ -14,7 +14,7 @@ "type": ["null", "string"] }, "score": { - "type": ["null", "string"] + "type": ["null", "number"] }, "recommendation": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json index 757084df0dfe7..fa6ba87dea343 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json @@ -1,4 +1,5 @@ { + "type": "object", "properties": { "id": { "type": ["null", "integer"] @@ -72,7 +73,7 @@ "type": ["null", "string"] }, "current_subtotal_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "current_subtotal_price_set": { "type": ["null", "object"], @@ -81,7 +82,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -92,7 +93,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -102,7 +103,7 @@ } }, "current_total_discounts": { - "type": ["null", "string"] + "type": ["null", "number"] }, "current_total_discounts_set": { "type": ["null", "object"], @@ -111,7 +112,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -122,7 +123,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -135,7 +136,7 @@ "type": ["null", "string"] }, "current_total_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "current_total_price_set": { "type": ["null", "object"], @@ -144,7 +145,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -155,7 +156,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -165,7 +166,7 @@ } }, "current_total_tax": { - "type": ["null", "string"] + "type": ["null", "number"] }, "current_total_tax_set": { "type": ["null", "object"], @@ -174,7 +175,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -185,7 +186,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -290,7 +291,7 @@ "type": ["null", "string"] }, "subtotal_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "subtotal_price_set": { "type": ["null", "object"], @@ -299,7 +300,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -310,7 +311,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -328,7 +329,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "rate": { "type": ["null", "number"] @@ -343,7 +344,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -354,7 +355,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -376,7 +377,7 @@ "type": ["null", "string"] }, "total_discounts": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_discounts_set": { "type": ["null", "object"], @@ -385,7 +386,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -396,7 +397,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -406,7 +407,7 @@ } }, "total_line_items_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_line_items_price_set": { "type": ["null", "object"], @@ -415,7 +416,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -426,7 +427,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -436,10 +437,10 @@ } }, "total_outstanding": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_price_set": { "type": ["null", "object"], @@ -448,7 +449,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -459,7 +460,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -469,7 +470,7 @@ } }, "total_price_usd": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_shipping_price_set": { "type": ["null", "object"], @@ -478,7 +479,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -489,7 +490,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -499,7 +500,7 @@ } }, "total_tax": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_tax_set": { "type": ["null", "object"], @@ -508,7 +509,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -519,7 +520,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -529,7 +530,7 @@ } }, "total_tip_received": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_weight": { "type": ["null", "integer"] @@ -624,7 +625,7 @@ "type": ["null", "string"] }, "total_spent": { - "type": ["null", "string"] + "type": ["null", "number"] }, "last_order_id": { "type": ["null", "integer"] @@ -869,7 +870,7 @@ } }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -878,7 +879,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -889,7 +890,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -923,7 +924,7 @@ "type": ["null", "string"] }, "total_discount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_discount_set": { "type": ["null", "object"], @@ -932,7 +933,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -943,7 +944,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -970,7 +971,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -979,7 +980,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -990,7 +991,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1232,7 +1233,7 @@ } }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -1241,7 +1242,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1252,7 +1253,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1286,7 +1287,7 @@ "type": ["null", "string"] }, "total_discount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_discount_set": { "type": ["null", "object"], @@ -1295,7 +1296,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1306,7 +1307,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1333,7 +1334,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -1342,7 +1343,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1353,7 +1354,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1595,7 +1596,7 @@ "type": ["null", "string"] }, "discounted_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "discounted_price_set": { "type": ["null", "object"], @@ -1604,7 +1605,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1615,7 +1616,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1628,7 +1629,7 @@ "type": ["null", "string"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -1637,7 +1638,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1648,7 +1649,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json index 0fb5b34d67b79..ac2f37095ef31 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json @@ -173,7 +173,7 @@ "type": ["null", "integer"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "image_id": { "type": ["null", "integer"] @@ -209,7 +209,7 @@ "type": ["null", "string"] }, "compare_at_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "updated_at": { "type": ["null", "string"], @@ -230,7 +230,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -238,7 +238,7 @@ } }, "compare_at_price": { - "type": ["null", "string"] + "type": ["null", "number"] } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json index c89db3250bf4d..32a9dfbb1503b 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json @@ -22,7 +22,7 @@ "type": ["null", "integer"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "authorization": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index eed41efeee80f..6b295857ee4a6 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -34,6 +34,7 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from .transform import DataTypeEnforcer from .utils import EagerlyCachedStreamState as stream_state_cache from .utils import ShopifyRateLimiter as limiter @@ -51,6 +52,7 @@ class ShopifyStream(HttpStream, ABC): def __init__(self, config: Dict): super().__init__(authenticator=config["authenticator"]) + self._transformer = DataTypeEnforcer(self.get_json_schema()) self.config = config @property @@ -78,7 +80,11 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: json_response = response.json() records = json_response.get(self.data_field, []) if self.data_field is not None else json_response - yield from records + # transform method was implemented according to issue 4841 + # Shopify API returns price fields as a string and it should be converted to number + # this solution designed to convert string into number, but in future can be modified for general purpose + for record in records: + yield self._transformer.transform(record) @property @abstractmethod diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py b/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py new file mode 100644 index 0000000000000..ad05dd5c0fa72 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py @@ -0,0 +1,121 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from decimal import Decimal +from typing import Any, Iterable, List, Mapping, MutableMapping + + +class DataTypeEnforcer: + """ + Transform class was implemented according to issue #4841 + Shopify API returns price fields as a string and it should be converted to the number + Some records fields contain objects and arrays, which contain price fields. + Those price fields should be transformed too. + This solution designed to convert string into number, but in future can be modified for general purpose + Correct types placed in schemes + Transformer iterates over records, compare values type with schema type and transform if it's needed + + Methods + ------- + _transform_array(self, array: List[Any], item_properties: Mapping[str, Any]) + Some fields type is array. Items inside array contain price fields, which should be transformed + This method iterate over items in array, compare schema types and convert if necessary + transform(self, field: Any, schema: Mapping[str, Any] = None) + Accepts field of Any type and schema, compere type of field and type in schema, convert if necessary + """ + + def __init__(self, schema: Mapping[str, Any], **kwargs): + super().__init__(**kwargs) + self._schema = schema + + @staticmethod + def _get_json_types(value_type: Any) -> List[str]: + json_types = { + str: ["string"], + int: ["integer", "number"], + float: ["number"], + dict: ["object"], + list: ["array"], + bool: ["boolean"], + type(None): [ + "null", + ], + } + return json_types.get(value_type) + + @staticmethod + def _types_from_schema(properties: Mapping[str, Any]) -> str: + schema_types = properties.get("type", []) + if not isinstance(schema_types, list): + schema_types = [ + schema_types, + ] + return schema_types + + @staticmethod + def _first_non_null_type(schema_types: List[str]) -> str: + not_null_types = schema_types.copy() + if "null" in not_null_types: + not_null_types.remove("null") + return not_null_types[0] + + @staticmethod + def _transform_number(value: Any): + return Decimal(value) + + def _transform_array(self, array: List[Any], item_properties: Mapping[str, Any]): + # iterate over items in array, compare schema types and convert if necessary. + for index, record in enumerate(array): + array[index] = self.transform(record, item_properties) + return array + + def _transform_object(self, record: MutableMapping[str, Any], properties: Mapping[str, Any]): + # compare schema types and convert if necessary. + for object_property, value in record.items(): + if value is None: + continue + if object_property in properties: + object_properties = properties.get(object_property) or {} + record[object_property] = self.transform(value, object_properties) + return record + + def transform(self, field: Any, schema: Mapping[str, Any] = None) -> Iterable[MutableMapping]: + schema = schema if schema is not None else self._schema + # get available types from schema + schema_types = self._types_from_schema(schema) + if schema_types and field is not None: + # if types presented in schema and field is not None, get available JSON Schema types for field + # and not null types from schema, check if field JSON Schema types presented in schema + field_json_types = self._get_json_types(type(field)) + schema_type = self._first_non_null_type(schema_types) + if not any(field_json_type in schema_types for field_json_type in field_json_types): + if schema_type == "number": + return self._transform_number(field) + if schema_type == "object": + properties = schema.get("properties", {}) + return self._transform_object(field, properties) + if schema_type == "array": + properties = schema.get("items", {}) + return self._transform_array(field, properties) + return field diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py new file mode 100644 index 0000000000000..7f2e52af8855b --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py @@ -0,0 +1,277 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from decimal import Decimal + +import pytest +from source_shopify.transform import DataTypeEnforcer + + +def find_by_path(path_list, value): + key_or_index = path_list.pop(0) + result_value = value[key_or_index] + if len(path_list) > 0: + result_value = find_by_path(path_list, result_value) + return result_value + + +def run_check(transform_object, schema, checks): + transformer = DataTypeEnforcer(schema) + transform_object = transformer.transform(transform_object) + for check in checks: + expected_value = find_by_path(check.get("path"), transform_object) + assert isinstance(expected_value, check["expected_type"]) + + +@pytest.mark.parametrize( + "transform_object, schema, checks", + [ + ( + {"id": 1}, + { + "type": "object", + "properties": { + "total_price": {"type": ["null", "integer"]}, + }, + }, + [ + { + "path": [ + "id", + ], + "expected_type": int, + }, + ], + ), + ( + { + "created_at": "2021-07-08T04:58:50-07:00", + }, + { + "type": "object", + "properties": { + "total_price": {"type": ["null", "string"]}, + }, + }, + [ + { + "path": [ + "created_at", + ], + "expected_type": str, + }, + ], + ), + ( + { + "note": None, + }, + { + "type": "object", + "properties": { + "total_price": {"type": ["null", "string"]}, + }, + }, + [ + { + "path": [ + "note", + ], + "expected_type": type(None), + }, + ], + ), + ( + { + "buyer_accepts_marketing": False, + }, + { + "type": "object", + "properties": { + "total_price": {"type": ["null", "boolean"]}, + }, + }, + [ + { + "path": [ + "buyer_accepts_marketing", + ], + "expected_type": bool, + }, + ], + ), + ], +) +def test_enforcer_correct_type(transform_object, schema, checks): + run_check(transform_object, schema, checks) + + +@pytest.mark.parametrize( + "transform_object, schema, checks", + [ + ( + { + "total_discounts": "0.00", + }, + { + "type": "object", + "properties": { + "total_discounts": {"type": ["null", "number"]}, + }, + }, + [ + { + "path": [ + "total_discounts", + ], + "expected_type": Decimal, + }, + ], + ), + ( + { + "total_price": "39.17", + }, + { + "type": "object", + "properties": { + "total_price": {"type": ["null", "number"]}, + }, + }, + [ + { + "path": [ + "total_price", + ], + "expected_type": Decimal, + }, + ], + ), + ], +) +def test_enforcer_string_to_number(transform_object, schema, checks): + run_check(transform_object, schema, checks) + + +@pytest.mark.parametrize( + "transform_object, schema, checks", + [ + ( + { + "customer": {"total_spent": "0.00"}, + }, + { + "type": "object", + "properties": { + "customer": {"type": "object", "properties": {"total_spent": {"type": ["null", "number"]}}}, + }, + }, + [ + {"path": ["customer", "total_spent"], "expected_type": Decimal}, + ], + ) + ], +) +def test_enforcer_nested_object(transform_object, schema, checks): + run_check(transform_object, schema, checks) + + +@pytest.mark.parametrize( + "transform_object, schema, checks", + [ + ( + { + "shipping_lines": [{"price": "20.17"}], + }, + { + "type": "object", + "properties": { + "shipping_lines": { + "items": {"properties": {"price": {"type": ["null", "number"]}}, "type": ["null", "object"]}, + "type": ["null", "array"], + }, + }, + }, + [ + {"path": ["shipping_lines", 0, "price"], "expected_type": Decimal}, + ], + ), + ( + { + "line_items": [{"gift_card": False, "grams": 112, "line_price": "19.00", "price": "19.00"}], + }, + { + "type": "object", + "properties": { + "line_items": { + "items": { + "properties": { + "grams": {"type": ["null", "integer"]}, + "line_price": {"type": ["null", "number"]}, + "gift_card": {"type": ["null", "boolean"]}, + "price": {"type": ["null", "number"]}, + }, + "type": ["null", "object"], + }, + "type": ["null", "array"], + }, + }, + }, + [ + {"path": ["line_items", 0, "line_price"], "expected_type": Decimal}, + ], + ), + ], +) +def test_enforcer_nested_array(transform_object, schema, checks): + run_check(transform_object, schema, checks) + + +@pytest.mark.parametrize( + "transform_object, schema, checks", + [ + ( + { + "line_items": ["19.00", "0.00"], + }, + { + "type": "object", + "properties": { + "line_items": { + "items": { + "type": ["null", "number"], + }, + "type": ["null", "array"], + }, + }, + }, + [ + {"path": ["line_items", 0], "expected_type": Decimal}, + {"path": ["line_items", 1], "expected_type": Decimal}, + ], + ), + ], +) +def test_enforcer_string_to_number_in_array(transform_object, schema, checks): + run_check(transform_object, schema, checks) diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md index a263edbba78ff..8faea2ec035d6 100644 --- a/docs/integrations/sources/shopify.md +++ b/docs/integrations/sources/shopify.md @@ -67,6 +67,7 @@ With given error message the sync operation is still goes on, but will require m | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.1.17 | 2021-09-17 | [5244](https://github.com/airbytehq/airbyte/pull/5244) | Created data type enforcer for converting prices into numbers | | 0.1.16 | 2021-09-09 | [5965](https://github.com/airbytehq/airbyte/pull/5945) | Fixed the connector's performance for `Incremental refresh` | | 0.1.15 | 2021-09-02 | [5853](https://github.com/airbytehq/airbyte/pull/5853) | Fixed `amount` type in `order_refund` schema | | 0.1.14 | 2021-09-02 | [5801](https://github.com/airbytehq/airbyte/pull/5801) | Fixed `line_items/discount allocations` & `duties` parts of `orders` schema |