diff --git a/airbyte-integrations/connectors/source-shipstation/.dockerignore b/airbyte-integrations/connectors/source-shipstation/.dockerignore new file mode 100644 index 0000000000000..aba57c42cf5e7 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_shipstation +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-shipstation/Dockerfile b/airbyte-integrations/connectors/source-shipstation/Dockerfile new file mode 100644 index 0000000000000..0676d312db081 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_shipstation ./source_shipstation + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-shipstation diff --git a/airbyte-integrations/connectors/source-shipstation/README.md b/airbyte-integrations/connectors/source-shipstation/README.md new file mode 100644 index 0000000000000..6fd20b2b6bd57 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/README.md @@ -0,0 +1,132 @@ +# Shipstation Source + +This is the repository for the Shipstation source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://help.shipstation.com/hc/en-us/articles/360025856212-ShipStation-API). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-shipstation:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/shipstation) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_shipstation/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source shipstation test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-shipstation:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-shipstation:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-shipstation:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shipstation:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shipstation:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-shipstation:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-shipstation:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-shipstation:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-shipstation/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shipstation/acceptance-test-config.yml new file mode 100644 index 0000000000000..c0fd5261472e7 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-shipstation:dev +tests: + spec: + - spec_path: "source_shipstation/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-shipstation/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-shipstation/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-shipstation/build.gradle b/airbyte-integrations/connectors/source-shipstation/build.gradle new file mode 100644 index 0000000000000..88d61a605215f --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_shipstation' +} diff --git a/airbyte-integrations/connectors/source-shipstation/integration_tests/__init__.py b/airbyte-integrations/connectors/source-shipstation/integration_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-shipstation/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shipstation/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/airbyte-integrations/connectors/source-shipstation/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-shipstation/integration_tests/acceptance.py new file mode 100644 index 0000000000000..056971f954502 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-shipstation/integration_tests/catalog.json b/airbyte-integrations/connectors/source-shipstation/integration_tests/catalog.json new file mode 100644 index 0000000000000..9990e529eccbc --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/integration_tests/catalog.json @@ -0,0 +1,1127 @@ +{ + "streams": [ + { + "name": "customers", + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": true, + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "customerId": { + "type": "integer" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + }, + "name": { + "type": "string" + }, + "company": { + "type": "string" + }, + "street1": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "countryCode": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "email": { + "type": "string" + }, + "addressVerified": { + "type": "string" + }, + "marketplaceUsernames": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "customerUserId": { + "type": "integer" + }, + "customerId": { + "type": "integer" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + }, + "marketplaceId": { + "type": "integer" + }, + "marketplace": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "customerUserId", + "customerId", + "createDate", + "modifyDate", + "marketplaceId", + "marketplace", + "username" + ] + } + ], + "additionalItems": false + }, + "tags": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "tagId": { + "type": "integer" + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "tagId", + "name" + ] + } + ], + "additionalItems": false + } + }, + "additionalProperties": false + } + }, + { + "name": "fulfillments", + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "fulfillmentId": { + "type": "integer" + }, + "orderId": { + "type": "integer" + }, + "orderNumber": { + "type": "string" + }, + "userId": { + "type": "string" + }, + "customerEmail": { + "type": "string" + }, + "trackingNumber": { + "type": "string" + }, + "createDate": { + "type": "string" + }, + "shipDate": { + "type": "string" + }, + "voidDate": { + "type": "null" + }, + "deliveryDate": { + "type": "null" + }, + "carrierCode": { + "type": "string" + }, + "fulfillmentProviderCode": { + "type": "null" + }, + "fulfillmentServiceCode": { + "type": "null" + }, + "fulfillmentFee": { + "type": "integer" + }, + "voidRequested": { + "type": "boolean" + }, + "voided": { + "type": "boolean" + }, + "marketplaceNotified": { + "type": "boolean" + }, + "notifyErrorMessage": { + "type": "null" + }, + "shipTo": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "company": { + "type": "null" + }, + "street1": { + "type": "string" + }, + "street2": { + "type": "null" + }, + "street3": { + "type": "null" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "country": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "residential": { + "type": "null" + }, + "addressVerified": { + "type": "null" + } + }, + "required": [] + } + }, + "required": [] + } + }, + { + "name": "orders", + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "orderId": { + "type": "integer" + }, + "orderNumber": { + "type": "string" + }, + "orderKey": { + "type": "string" + }, + "orderDate": { + "type": "string" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + }, + "paymentDate": { + "type": "string" + }, + "shipByDate": { + "type": "string" + }, + "orderStatus": { + "type": "string" + }, + "customerId": { + "type": "integer" + }, + "customerUsername": { + "type": "string" + }, + "customerEmail": { + "type": "string" + }, + "billTo": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "company": { + "type": "null" + }, + "street1": { + "type": "null" + }, + "street2": { + "type": "null" + }, + "street3": { + "type": "null" + }, + "city": { + "type": "null" + }, + "state": { + "type": "null" + }, + "postalCode": { + "type": "null" + }, + "country": { + "type": "null" + }, + "phone": { + "type": "null" + }, + "residential": { + "type": "null" + }, + "addressVerified": { + "type": "null" + } + }, + "required": [ + "name", + "company", + "street1", + "street2", + "street3", + "city", + "state", + "postalCode", + "country", + "phone", + "residential", + "addressVerified" + ] + }, + "shipTo": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "company": { + "type": "string" + }, + "street1": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "street3": { + "type": "null" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "country": { + "type": "string" + }, + "phone": { + "type": "null" + }, + "residential": { + "type": "boolean" + }, + "addressVerified": { + "type": "string" + } + }, + "required": [ + "name", + "company", + "street1", + "street2", + "street3", + "city", + "state", + "postalCode", + "country", + "phone", + "residential", + "addressVerified" + ] + }, + "items": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "orderItemId": { + "type": "integer" + }, + "lineItemKey": { + "type": "null" + }, + "sku": { + "type": "string" + }, + "name": { + "type": "string" + }, + "imageUrl": { + "type": "null" + }, + "weight": { + "type": "object", + "properties": { + "value": { + "type": "number" + }, + "units": { + "type": "string" + } + }, + "required": [ + "value", + "units" + ] + }, + "quantity": { + "type": "integer" + }, + "unitPrice": { + "type": "number" + }, + "taxAmount": { + "type": "number" + }, + "shippingAmount": { + "type": "number" + }, + "warehouseLocation": { + "type": "string" + }, + "options": { + "type": "array", + "items": {} + }, + "productId": { + "type": "integer" + }, + "fulfillmentSku": { + "type": "string" + }, + "adjustment": { + "type": "boolean" + }, + "upc": { + "type": "null" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + } + }, + "required": [ + "orderItemId", + "lineItemKey", + "sku", + "name", + "imageUrl", + "weight", + "quantity", + "unitPrice", + "taxAmount", + "shippingAmount", + "warehouseLocation", + "options", + "productId", + "fulfillmentSku", + "adjustment", + "upc", + "createDate", + "modifyDate" + ] + } + ] + }, + "orderTotal": { + "type": "number" + }, + "amountPaid": { + "type": "number" + }, + "taxAmount": { + "type": "number" + }, + "shippingAmount": { + "type": "number" + }, + "customerNotes": { + "type": "string" + }, + "internalNotes": { + "type": "string" + }, + "gift": { + "type": "boolean" + }, + "giftMessage": { + "type": "null" + }, + "paymentMethod": { + "type": "null" + }, + "requestedShippingService": { + "type": "string" + }, + "carrierCode": { + "type": "string" + }, + "serviceCode": { + "type": "string" + }, + "packageCode": { + "type": "string" + }, + "confirmation": { + "type": "string" + }, + "shipDate": { + "type": "string" + }, + "holdUntilDate": { + "type": "null" + }, + "weight": { + "type": "object", + "properties": { + "value": { + "type": "number" + }, + "units": { + "type": "string" + } + }, + "required": [ + "value", + "units" + ] + }, + "dimensions": { + "type": [ + "object", + "null" + ], + "properties": { + "units": { + "type": "string" + }, + "length": { + "type": "number" + }, + "width": { + "type": "number" + }, + "height": { + "type": "number" + } + }, + "required": [ + "units", + "length", + "width", + "height" + ] + }, + "insuranceOptions": { + "type": "object", + "properties": { + "provider": { + "type": "null" + }, + "insureShipment": { + "type": "boolean" + }, + "insuredValue": { + "type": "number" + } + }, + "required": [ + "provider", + "insureShipment", + "insuredValue" + ] + }, + "internationalOptions": { + "type": "object", + "properties": { + "contents": { + "type": "string" + }, + "customsItems": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "customsItemId": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "quantity": { + "type": "integer" + }, + "value": { + "type": "number" + }, + "harmonizedTariffCode": { + "type": "null" + }, + "countryOfOrigin": { + "type": "string" + } + }, + "required": [ + "customsItemId", + "description", + "quantity", + "value", + "harmonizedTariffCode", + "countryOfOrigin" + ] + } + ] + }, + "nonDelivery": { + "type": "string" + } + }, + "required": [ + "contents", + "customsItems", + "nonDelivery" + ] + }, + "advancedOptions": { + "type": "object", + "properties": { + "warehouseId": { + "type": "integer" + }, + "nonMachinable": { + "type": "boolean" + }, + "saturdayDelivery": { + "type": "boolean" + }, + "containsAlcohol": { + "type": "boolean" + }, + "mergedOrSplit": { + "type": "boolean" + }, + "mergedIds": { + "type": "array", + "items": {} + }, + "parentId": { + "type": "null" + }, + "storeId": { + "type": "integer" + }, + "customField1": { + "type": "string" + }, + "customField2": { + "type": "string" + }, + "customField3": { + "type": "null" + }, + "source": { + "type": "null" + }, + "billToParty": { + "type": "null" + }, + "billToAccount": { + "type": "null" + }, + "billToPostalCode": { + "type": "null" + }, + "billToCountryCode": { + "type": "null" + } + }, + "required": [ + "warehouseId", + "nonMachinable", + "saturdayDelivery", + "containsAlcohol", + "mergedOrSplit", + "mergedIds", + "parentId", + "storeId", + "customField1", + "customField2", + "customField3", + "source", + "billToParty", + "billToAccount", + "billToPostalCode", + "billToCountryCode" + ] + }, + "tagIds": { + "type": "null" + }, + "userId": { + "type": "null" + }, + "externallyFulfilled": { + "type": "boolean" + }, + "externallyFulfilledBy": { + "type": "null" + } + }, + "required": [ + "orderId", + "orderNumber", + "orderKey", + "orderDate", + "createDate", + "modifyDate", + "paymentDate", + "shipByDate", + "orderStatus", + "customerId", + "customerUsername", + "customerEmail", + "billTo", + "shipTo", + "items", + "orderTotal", + "amountPaid", + "taxAmount", + "shippingAmount", + "customerNotes", + "internalNotes", + "gift", + "giftMessage", + "paymentMethod", + "requestedShippingService", + "carrierCode", + "serviceCode", + "packageCode", + "confirmation", + "shipDate", + "holdUntilDate", + "weight", + "dimensions", + "insuranceOptions", + "internationalOptions", + "advancedOptions", + "tagIds", + "userId", + "externallyFulfilled", + "externallyFulfilledBy" + ] + } + }, + { + "name": "stores", + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "storeId": { + "type": "integer" + }, + "storeName": { + "type": "string" + }, + "marketplaceId": { + "type": "integer" + }, + "marketplaceName": { + "type": "string" + }, + "accountName": { + "type": "string" + }, + "email": { + "type": "string" + }, + "integrationUrl": { + "type": "string" + }, + "active": { + "type": "boolean" + }, + "companyName": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "publicEmail": { + "type": "string" + }, + "website": { + "type": "string" + }, + "refreshDate": { + "type": "string" + }, + "lastRefreshAttempt": { + "type": "string" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + }, + "autoRefresh": { + "type": "boolean" + }, + "statusMappings": { + "type": "string" + } + }, + "required": [] + } + }, + { + "name": "warehouses", + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "warehouseId": { + "type": "integer" + }, + "warehouseName": { + "type": "string" + }, + "originAddress": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "company": { + "type": "string" + }, + "street1": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "street3": { + "type": "string" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "country": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "residential": { + "type": "boolean" + }, + "addressVerified": { + "type": "string" + } + }, + "required": [] + }, + "returnAddress": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "company": { + "type": "string" + }, + "street1": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "street3": { + "type": "string" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "country": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "residential": { + "type": "string" + }, + "addressVerified": { + "type": "string" + } + }, + "required": [ + "name", + "company", + "street1", + "street2", + "street3", + "city", + "state", + "postalCode", + "country", + "phone", + "residential", + "addressVerified" + ] + }, + "createDate": { + "type": "string" + }, + "isDefault": { + "type": "boolean" + }, + "sellerIntegrationId": { + "type": "string" + }, + "extInventoryIdentity": { + "type": "string" + }, + "registerFedexMeter": { + "type": "string" + } + }, + "required": [] + } + }, + { + "name": "products", + "supported_sync_modes": [ + "full_refresh" + ], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "aliases": { + "type": [ + "string", + "null" + ] + }, + "productId": { + "type": "integer" + }, + "sku": { + "type": "string" + }, + "name": { + "type": "string" + }, + "price": { + "type": "number" + }, + "defaultCost": { + "type": "integer" + }, + "length": { + "type": [ + "number", + "null" + ] + }, + "width": { + "type": [ + "number", + "null" + ] + }, + "height": { + "type": [ + "number", + "null" + ] + }, + "weightOz": { + "type": [ + "number", + "null" + ] + }, + "internalNotes": { + "type": "null" + }, + "fulfillmentSku": { + "type": "string" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + }, + "active": { + "type": "boolean" + }, + "productCategory": { + "type": "object", + "properties": { + "categoryId": { + "type": "integer" + }, + "name": { + "type": "string" + } + }, + "required": [] + }, + "productType": { + "type": "null" + }, + "warehouseLocation": { + "type": "string" + }, + "defaultCarrierCode": { + "type": "string" + }, + "defaultServiceCode": { + "type": "string" + }, + "defaultPackageCode": { + "type": "string" + }, + "defaultIntlCarrierCode": { + "type": "string" + }, + "defaultIntlServiceCode": { + "type": "string" + }, + "defaultIntlPackageCode": { + "type": "string" + }, + "defaultConfirmation": { + "type": "string" + }, + "defaultIntlConfirmation": { + "type": "string" + }, + "customsDescription": { + "type": "null" + }, + "customsValue": { + "type": "null" + }, + "customsTariffNo": { + "type": "null" + }, + "customsCountryCode": { + "type": "null" + }, + "noCustoms": { + "type": "null" + }, + "tags": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "tagId": { + "type": "integer" + }, + "name": { + "type": "string" + } + }, + "required": [ + "tagId", + "name" + ] + } + ] + } + }, + "required": [] + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-shipstation/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shipstation/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..3daaa9321bc50 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/integration_tests/configured_catalog.json @@ -0,0 +1,58 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "fulfillments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "products", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "orders", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "stores", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "warehouses", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-shipstation/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-shipstation/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..13ab2cb892772 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "56sd4a65d4sa65d4as65d4", + "api_secret": "5a6sd4as65d4a6s5d465as4", +} diff --git a/airbyte-integrations/connectors/source-shipstation/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-shipstation/integration_tests/sample_config.json new file mode 100644 index 0000000000000..8385b74be85ae --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "", + "api_secret": "" +} diff --git a/airbyte-integrations/connectors/source-shipstation/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-shipstation/integration_tests/sample_state.json new file mode 100644 index 0000000000000..3587e579822d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-shipstation/main.py b/airbyte-integrations/connectors/source-shipstation/main.py new file mode 100644 index 0000000000000..f3cd8679a9ef4 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_shipstation import SourceShipstation + +if __name__ == "__main__": + source = SourceShipstation() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-shipstation/requirements.txt b/airbyte-integrations/connectors/source-shipstation/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-shipstation/setup.py b/airbyte-integrations/connectors/source-shipstation/setup.py new file mode 100644 index 0000000000000..a8b9b5cac952d --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_shipstation", + description="Source implementation for Shipstation.", + author="Ehmad Zubair", + author_email="ehmadz@cogentlabs.co", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-shipstation/source_shipstation/__init__.py b/airbyte-integrations/connectors/source-shipstation/source_shipstation/__init__.py new file mode 100644 index 0000000000000..1b828592c1d2a --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/source_shipstation/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceShipstation + +__all__ = ["SourceShipstation"] diff --git a/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/customers.json b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/customers.json new file mode 100644 index 0000000000000..fe07fd3f7db1d --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/customers.json @@ -0,0 +1,113 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "customerId": { + "type": "integer" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + }, + "name": { + "type": "string" + }, + "company": { + "type": "string" + }, + "street1": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "countryCode": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "email": { + "type": "string" + }, + "addressVerified": { + "type": "string" + }, + "marketplaceUsernames": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "customerUserId": { + "type": "integer" + }, + "customerId": { + "type": "integer" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + }, + "marketplaceId": { + "type": "integer" + }, + "marketplace": { + "type": "string" + }, + "username": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "customerUserId", + "customerId", + "createDate", + "modifyDate", + "marketplaceId", + "marketplace", + "username" + ] + } + ], + "additionalItems": false + }, + "tags": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "tagId": { + "type": "integer" + }, + "name": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "tagId", + "name" + ] + } + ], + "additionalItems": false + } + }, + "additionalProperties": false +} diff --git a/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/employees.json b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/employees.json new file mode 100644 index 0000000000000..2fa01a0fa1ff9 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/employees.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/fulfillments.json b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/fulfillments.json new file mode 100644 index 0000000000000..f948640e28d00 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/fulfillments.json @@ -0,0 +1,103 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "fulfillmentId": { + "type": "integer" + }, + "orderId": { + "type": "integer" + }, + "orderNumber": { + "type": "string" + }, + "userId": { + "type": "string" + }, + "customerEmail": { + "type": "string" + }, + "trackingNumber": { + "type": "string" + }, + "createDate": { + "type": "string" + }, + "shipDate": { + "type": "string" + }, + "voidDate": { + "type": "null" + }, + "deliveryDate": { + "type": "null" + }, + "carrierCode": { + "type": "string" + }, + "fulfillmentProviderCode": { + "type": "null" + }, + "fulfillmentServiceCode": { + "type": "null" + }, + "fulfillmentFee": { + "type": "integer" + }, + "voidRequested": { + "type": "boolean" + }, + "voided": { + "type": "boolean" + }, + "marketplaceNotified": { + "type": "boolean" + }, + "notifyErrorMessage": { + "type": "null" + }, + "shipTo": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "company": { + "type": "null" + }, + "street1": { + "type": "string" + }, + "street2": { + "type": "null" + }, + "street3": { + "type": "null" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "country": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "residential": { + "type": "null" + }, + "addressVerified": { + "type": "null" + } + }, + "required": [] + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/orders.json b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/orders.json new file mode 100644 index 0000000000000..eb91f3aadfa8a --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/orders.json @@ -0,0 +1,532 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "orderId": { + "type": "integer" + }, + "orderNumber": { + "type": "string" + }, + "orderKey": { + "type": "string" + }, + "orderDate": { + "type": "string" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + }, + "paymentDate": { + "type": "string" + }, + "shipByDate": { + "type": "string" + }, + "orderStatus": { + "type": "string" + }, + "customerId": { + "type": "integer" + }, + "customerUsername": { + "type": "string" + }, + "customerEmail": { + "type": "string" + }, + "billTo": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "company": { + "type": "null" + }, + "street1": { + "type": "null" + }, + "street2": { + "type": "null" + }, + "street3": { + "type": "null" + }, + "city": { + "type": "null" + }, + "state": { + "type": "null" + }, + "postalCode": { + "type": "null" + }, + "country": { + "type": "null" + }, + "phone": { + "type": "null" + }, + "residential": { + "type": "null" + }, + "addressVerified": { + "type": "null" + } + }, + "required": [ + "name", + "company", + "street1", + "street2", + "street3", + "city", + "state", + "postalCode", + "country", + "phone", + "residential", + "addressVerified" + ] + }, + "shipTo": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "company": { + "type": "string" + }, + "street1": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "street3": { + "type": "null" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "country": { + "type": "string" + }, + "phone": { + "type": "null" + }, + "residential": { + "type": "boolean" + }, + "addressVerified": { + "type": "string" + } + }, + "required": [ + "name", + "company", + "street1", + "street2", + "street3", + "city", + "state", + "postalCode", + "country", + "phone", + "residential", + "addressVerified" + ] + }, + "items": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "orderItemId": { + "type": "integer" + }, + "lineItemKey": { + "type": "null" + }, + "sku": { + "type": "string" + }, + "name": { + "type": "string" + }, + "imageUrl": { + "type": "null" + }, + "weight": { + "type": "object", + "properties": { + "value": { + "type": "number" + }, + "units": { + "type": "string" + } + }, + "required": [ + "value", + "units" + ] + }, + "quantity": { + "type": "integer" + }, + "unitPrice": { + "type": "number" + }, + "taxAmount": { + "type": "number" + }, + "shippingAmount": { + "type": "number" + }, + "warehouseLocation": { + "type": "string" + }, + "options": { + "type": "array", + "items": {} + }, + "productId": { + "type": "integer" + }, + "fulfillmentSku": { + "type": "string" + }, + "adjustment": { + "type": "boolean" + }, + "upc": { + "type": "null" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + } + }, + "required": [ + "orderItemId", + "lineItemKey", + "sku", + "name", + "imageUrl", + "weight", + "quantity", + "unitPrice", + "taxAmount", + "shippingAmount", + "warehouseLocation", + "options", + "productId", + "fulfillmentSku", + "adjustment", + "upc", + "createDate", + "modifyDate" + ] + } + ] + }, + "orderTotal": { + "type": "number" + }, + "amountPaid": { + "type": "number" + }, + "taxAmount": { + "type": "number" + }, + "shippingAmount": { + "type": "number" + }, + "customerNotes": { + "type": "string" + }, + "internalNotes": { + "type": "string" + }, + "gift": { + "type": "boolean" + }, + "giftMessage": { + "type": "null" + }, + "paymentMethod": { + "type": "null" + }, + "requestedShippingService": { + "type": "string" + }, + "carrierCode": { + "type": "string" + }, + "serviceCode": { + "type": "string" + }, + "packageCode": { + "type": "string" + }, + "confirmation": { + "type": "string" + }, + "shipDate": { + "type": "string" + }, + "holdUntilDate": { + "type": "null" + }, + "weight": { + "type": "object", + "properties": { + "value": { + "type": "number" + }, + "units": { + "type": "string" + } + }, + "required": [ + "value", + "units" + ] + }, + "dimensions": { + "type": ["object", "null"], + "properties": { + "units": { + "type": "string" + }, + "length": { + "type": "number" + }, + "width": { + "type": "number" + }, + "height": { + "type": "number" + } + }, + "required": [ + "units", + "length", + "width", + "height" + ] + }, + "insuranceOptions": { + "type": "object", + "properties": { + "provider": { + "type": "null" + }, + "insureShipment": { + "type": "boolean" + }, + "insuredValue": { + "type": "number" + } + }, + "required": [ + "provider", + "insureShipment", + "insuredValue" + ] + }, + "internationalOptions": { + "type": "object", + "properties": { + "contents": { + "type": "string" + }, + "customsItems": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "customsItemId": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "quantity": { + "type": "integer" + }, + "value": { + "type": "number" + }, + "harmonizedTariffCode": { + "type": "null" + }, + "countryOfOrigin": { + "type": "string" + } + }, + "required": [ + "customsItemId", + "description", + "quantity", + "value", + "harmonizedTariffCode", + "countryOfOrigin" + ] + } + ] + }, + "nonDelivery": { + "type": "string" + } + }, + "required": [ + "contents", + "customsItems", + "nonDelivery" + ] + }, + "advancedOptions": { + "type": "object", + "properties": { + "warehouseId": { + "type": "integer" + }, + "nonMachinable": { + "type": "boolean" + }, + "saturdayDelivery": { + "type": "boolean" + }, + "containsAlcohol": { + "type": "boolean" + }, + "mergedOrSplit": { + "type": "boolean" + }, + "mergedIds": { + "type": "array", + "items": {} + }, + "parentId": { + "type": "null" + }, + "storeId": { + "type": "integer" + }, + "customField1": { + "type": "string" + }, + "customField2": { + "type": "string" + }, + "customField3": { + "type": "null" + }, + "source": { + "type": "null" + }, + "billToParty": { + "type": "null" + }, + "billToAccount": { + "type": "null" + }, + "billToPostalCode": { + "type": "null" + }, + "billToCountryCode": { + "type": "null" + } + }, + "required": [ + "warehouseId", + "nonMachinable", + "saturdayDelivery", + "containsAlcohol", + "mergedOrSplit", + "mergedIds", + "parentId", + "storeId", + "customField1", + "customField2", + "customField3", + "source", + "billToParty", + "billToAccount", + "billToPostalCode", + "billToCountryCode" + ] + }, + "tagIds": { + "type": "null" + }, + "userId": { + "type": "null" + }, + "externallyFulfilled": { + "type": "boolean" + }, + "externallyFulfilledBy": { + "type": "null" + } + }, + "required": [ + "orderId", + "orderNumber", + "orderKey", + "orderDate", + "createDate", + "modifyDate", + "paymentDate", + "shipByDate", + "orderStatus", + "customerId", + "customerUsername", + "customerEmail", + "billTo", + "shipTo", + "items", + "orderTotal", + "amountPaid", + "taxAmount", + "shippingAmount", + "customerNotes", + "internalNotes", + "gift", + "giftMessage", + "paymentMethod", + "requestedShippingService", + "carrierCode", + "serviceCode", + "packageCode", + "confirmation", + "shipDate", + "holdUntilDate", + "weight", + "dimensions", + "insuranceOptions", + "internationalOptions", + "advancedOptions", + "tagIds", + "userId", + "externallyFulfilled", + "externallyFulfilledBy" + ] +} diff --git a/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/products.json b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/products.json new file mode 100644 index 0000000000000..f1e504c911dc5 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/products.json @@ -0,0 +1,129 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "aliases": { + "type": ["string", "null"] + }, + "productId": { + "type": "integer" + }, + "sku": { + "type": "string" + }, + "name": { + "type": "string" + }, + "price": { + "type": "number" + }, + "defaultCost": { + "type": "integer" + }, + "length": { + "type": ["number", "null"] + }, + "width": { + "type": ["number", "null"] + }, + "height": { + "type": ["number", "null"] + }, + "weightOz": { + "type": ["number", "null"] + }, + "internalNotes": { + "type": "null" + }, + "fulfillmentSku": { + "type": "string" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + }, + "active": { + "type": "boolean" + }, + "productCategory": { + "type": "object", + "properties": { + "categoryId": { + "type": "integer" + }, + "name": { + "type": "string" + } + }, + "required": [] + }, + "productType": { + "type": "null" + }, + "warehouseLocation": { + "type": "string" + }, + "defaultCarrierCode": { + "type": "string" + }, + "defaultServiceCode": { + "type": "string" + }, + "defaultPackageCode": { + "type": "string" + }, + "defaultIntlCarrierCode": { + "type": "string" + }, + "defaultIntlServiceCode": { + "type": "string" + }, + "defaultIntlPackageCode": { + "type": "string" + }, + "defaultConfirmation": { + "type": "string" + }, + "defaultIntlConfirmation": { + "type": "string" + }, + "customsDescription": { + "type": "null" + }, + "customsValue": { + "type": "null" + }, + "customsTariffNo": { + "type": "null" + }, + "customsCountryCode": { + "type": "null" + }, + "noCustoms": { + "type": "null" + }, + "tags": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "tagId": { + "type": "integer" + }, + "name": { + "type": "string" + } + }, + "required": [ + "tagId", + "name" + ] + } + ] + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/stores.json b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/stores.json new file mode 100644 index 0000000000000..c69695cd3682f --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/stores.json @@ -0,0 +1,61 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "storeId": { + "type": "integer" + }, + "storeName": { + "type": "string" + }, + "marketplaceId": { + "type": "integer" + }, + "marketplaceName": { + "type": "string" + }, + "accountName": { + "type": "string" + }, + "email": { + "type": "string" + }, + "integrationUrl": { + "type": "string" + }, + "active": { + "type": "boolean" + }, + "companyName": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "publicEmail": { + "type": "string" + }, + "website": { + "type": "string" + }, + "refreshDate": { + "type": "string" + }, + "lastRefreshAttempt": { + "type": "string" + }, + "createDate": { + "type": "string" + }, + "modifyDate": { + "type": "string" + }, + "autoRefresh": { + "type": "boolean" + }, + "statusMappings": { + "type": "string" + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/warehouses.json b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/warehouses.json new file mode 100644 index 0000000000000..dc6c0ee057a5b --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/source_shipstation/schemas/warehouses.json @@ -0,0 +1,125 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "warehouseId": { + "type": "integer" + }, + "warehouseName": { + "type": "string" + }, + "originAddress": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "company": { + "type": "string" + }, + "street1": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "street3": { + "type": "string" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "country": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "residential": { + "type": "boolean" + }, + "addressVerified": { + "type": "string" + } + }, + "required": [] + }, + "returnAddress": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "company": { + "type": "string" + }, + "street1": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "street3": { + "type": "string" + }, + "city": { + "type": "string" + }, + "state": { + "type": "string" + }, + "postalCode": { + "type": "string" + }, + "country": { + "type": "string" + }, + "phone": { + "type": "string" + }, + "residential": { + "type": "string" + }, + "addressVerified": { + "type": "string" + } + }, + "required": [ + "name", + "company", + "street1", + "street2", + "street3", + "city", + "state", + "postalCode", + "country", + "phone", + "residential", + "addressVerified" + ] + }, + "createDate": { + "type": "string" + }, + "isDefault": { + "type": "boolean" + }, + "sellerIntegrationId": { + "type": "string" + }, + "extInventoryIdentity": { + "type": "string" + }, + "registerFedexMeter": { + "type": "string" + } + }, + "required": [] +} diff --git a/airbyte-integrations/connectors/source-shipstation/source_shipstation/source.py b/airbyte-integrations/connectors/source-shipstation/source_shipstation/source.py new file mode 100644 index 0000000000000..bd210be4c93b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/source_shipstation/source.py @@ -0,0 +1,45 @@ +from typing import Any, List, Mapping, Tuple + +import requests +from requests.auth import HTTPBasicAuth +from requests.exceptions import HTTPError +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream + +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +from source_shipstation.streams import Customers, Fulfillments, Products, Orders, Stores, Warehouses + + +class SourceShipstation(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 + for an example. + + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + auth = HTTPBasicAuth(username=config["api_key"], password=config["api_secret"]) + try: + Fulfillments(authenticator=auth) + connection = True, None + except Exception as e: + connection = False, e + + return connection + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + auth = HTTPBasicAuth(username=config["api_key"], password=config['api_secret']) + return [ + Customers(authenticator=auth), + Fulfillments(authenticator=auth), + Products(authenticator=auth), + Orders(authenticator=auth), + Stores(authenticator=auth), + Warehouses(authenticator=auth) + ] diff --git a/airbyte-integrations/connectors/source-shipstation/source_shipstation/spec.json b/airbyte-integrations/connectors/source-shipstation/source_shipstation/spec.json new file mode 100644 index 0000000000000..c49d85634c04e --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/source_shipstation/spec.json @@ -0,0 +1,21 @@ +{ + "documentationUrl": "https://docsurl.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Shipstation Spec", + "type": "object", + "required": ["api_key", "api_secret"], + "additionalProperties": false, + "properties": { + "api_key": { + "type": "string", + "description": "The Shipstation username for Basic Authentication" + }, + "api_secret": { + "type": "string", + "description": "The Shipstation password for Basic Authentication", + "airbyte_secret": true + } + } + } +} diff --git a/airbyte-integrations/connectors/source-shipstation/source_shipstation/streams.py b/airbyte-integrations/connectors/source-shipstation/source_shipstation/streams.py new file mode 100644 index 0000000000000..abf0be0fbe53c --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/source_shipstation/streams.py @@ -0,0 +1,176 @@ +from abc import ABC +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +import requests + +from airbyte_cdk.sources.streams.http import HttpStream + + +class ShipstationStream(HttpStream, ABC): + url_base = "https://ssapi.shipstation.com" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + json_response = response.json() + total_pages = json_response['pages'] + current_page = json_response['page'] + if current_page < total_pages: + return {'next_page_num': current_page + 1} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + Usually contains common params e.g. pagination size etc. + """ + params = {'pageSize': 500} + + if next_page_token: + params.update({'page': next_page_token['next_page_num']}) + + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + :return an iterable containing each record in the response + """ + yield {} + + +class Customers(ShipstationStream): + primary_key = "customerId" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "/customers" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json()['customers'] + + +class Fulfillments(ShipstationStream): + primary_key = "fulfillmentId" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "/fulfillments" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json()['fulfillments'] + + +class Products(ShipstationStream): + primary_key = "productId" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "/products" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json()['products'] + + +class Orders(ShipstationStream): + primary_key = "orderId" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "/orders" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json()['orders'] + + +class Stores(ShipstationStream): + primary_key = "storeId" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "/stores" + + +class Warehouses(ShipstationStream): + primary_key = "warehouseId" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "/warehouses" + + +class IncrementalShipstationStream(ShipstationStream, ABC): + state_checkpoint_interval = None + + @property + def cursor_field(self) -> str: + """ + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return [] + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + return {} + + +class Employees(IncrementalShipstationStream): + + cursor_field = "start_date" + primary_key = "employee_id" + + def path(self, **kwargs) -> str: + """ + return "single". Required. + """ + return "employees" + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + """ + Slices control when state is saved. Specifically, state is saved after a slice has been fully read. + This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" + section of the docs for more information. + + The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the + necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. + This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. + + An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help + craft that specific request. + + For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement + this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date + till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into + the date query param. + """ + raise NotImplementedError("Implement stream slices or delete this method!") + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json() diff --git a/airbyte-integrations/connectors/source-shipstation/unit_tests/__init__.py b/airbyte-integrations/connectors/source-shipstation/unit_tests/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-shipstation/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-shipstation/unit_tests/test_incremental_streams.py new file mode 100644 index 0000000000000..68061bc8ec0bc --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_shipstation.source import IncrementalShipstationStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalShipstationStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalShipstationStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalShipstationStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalShipstationStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalShipstationStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalShipstationStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalShipstationStream, "cursor_field", "dummy_field") + stream = IncrementalShipstationStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalShipstationStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalShipstationStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-shipstation/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shipstation/unit_tests/test_source.py new file mode 100644 index 0000000000000..15a214956f266 --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_shipstation.source import SourceShipstation + + +def test_check_connection(mocker): + source = SourceShipstation() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceShipstation() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-shipstation/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-shipstation/unit_tests/test_streams.py new file mode 100644 index 0000000000000..0a56bdd01b41c --- /dev/null +++ b/airbyte-integrations/connectors/source-shipstation/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_shipstation.source import ShipstationStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(ShipstationStream, "path", "v0/example_endpoint") + mocker.patch.object(ShipstationStream, "primary_key", "test_primary_key") + mocker.patch.object(ShipstationStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = ShipstationStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = ShipstationStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = ShipstationStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = ShipstationStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = ShipstationStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = ShipstationStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = ShipstationStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time