Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SAT: check future_state only for incremental streams. #21248

Merged
merged 4 commits into from
Jan 13, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# Changelog

## 0.2.26
Check `future_state` only for incremental streams. [#21248](https://github.com/airbytehq/airbyte/pull/21248)

## 0.2.25
Enable bypass reason for future state test config.[#20549](https://github.com/airbytehq/airbyte/pull/20549)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./
COPY source_acceptance_test ./source_acceptance_test
RUN pip install .

LABEL io.airbyte.version=0.2.25
LABEL io.airbyte.version=0.2.26
LABEL io.airbyte.name=airbyte/source-acceptance-test

ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"]
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

import pendulum
import pytest
from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, AirbyteStateType, ConfiguredAirbyteCatalog, Type
from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, AirbyteStateType, ConfiguredAirbyteCatalog, SyncMode, Type
from source_acceptance_test import BaseTest
from source_acceptance_test.config import Config, EmptyStreamConfiguration, IncrementalConfig
from source_acceptance_test.utils import ConnectorRunner, JsonSchemaHelper, SecretDict, filter_output, incremental_only_catalog
Expand Down Expand Up @@ -40,7 +40,9 @@ def future_state_fixture(future_state_configuration, test_strictness_level, conf
if test_strictness_level is Config.TestStrictnessLevel.high:
if not all([missing_stream.bypass_reason is not None for missing_stream in missing_streams]):
pytest.fail("High test strictness level error: all missing_streams must have a bypass reason specified.")
all_stream_names = set([stream.stream.name for stream in configured_catalog.streams])
all_stream_names = {
stream.stream.name for stream in configured_catalog.streams if SyncMode.incremental in stream.stream.supported_sync_modes
}
streams_in_states = set([state["stream"]["stream_descriptor"]["name"] for state in states])
declared_missing_streams_names = set([missing_stream.name for missing_stream in missing_streams])
undeclared_missing_streams_names = all_stream_names - declared_missing_streams_names - streams_in_states
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#

import json
import time

import pytest
from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode
Expand Down Expand Up @@ -193,6 +194,8 @@ def test_connector_config_path_fixture(mocker, tmp_path, updated_configurations)
for configuration_file_name in updated_configurations:
updated_configuration_path = updated_configurations_dir / configuration_file_name
updated_configuration_path.touch()
# to avoid the equivalent 'ctime' for created files
time.sleep(0.01)

connector_config_path = conftest.connector_config_path_fixture.__wrapped__(inputs, base_path)
if not updated_configurations:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -744,18 +744,18 @@ def test_future_state_configuration_fixture(mocker, test_strictness_level, input
test_incremental.pytest.fail.assert_not_called()


TEST_AIRBYTE_STREAM_A = AirbyteStream(name="test_stream_a", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh])
TEST_AIRBYTE_STREAM_B = AirbyteStream(name="test_stream_b", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh])
TEST_AIRBYTE_STREAM_A = AirbyteStream(name="test_stream_a", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental])
TEST_AIRBYTE_STREAM_B = AirbyteStream(name="test_stream_b", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental])

TEST_CONFIGURED_AIRBYTE_STREAM_A = ConfiguredAirbyteStream(
stream=TEST_AIRBYTE_STREAM_A,
sync_mode=SyncMode.full_refresh,
sync_mode=SyncMode.incremental,
destination_sync_mode=DestinationSyncMode.overwrite,
)

TEST_CONFIGURED_AIRBYTE_STREAM_B = ConfiguredAirbyteStream(
stream=TEST_AIRBYTE_STREAM_B,
sync_mode=SyncMode.full_refresh,
sync_mode=SyncMode.incremental,
destination_sync_mode=DestinationSyncMode.overwrite,
)

Expand Down