Skip to content

Commit

Permalink
Delete now invalid tests
Browse files Browse the repository at this point in the history
  • Loading branch information
steinitzu committed Jun 27, 2024
1 parent a2b80d9 commit 0d0afa5
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 39 deletions.
38 changes: 0 additions & 38 deletions tests/load/pipeline/test_merge_disposition.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,44 +284,6 @@ def test_merge_no_merge_keys(destination_config: DestinationTestConfiguration) -
assert github_1_counts["issues"] == 100 - 45 + 10


@pytest.mark.parametrize(
"destination_config", destinations_configs(default_sql_configs=True), ids=lambda x: x.name
)
def test_merge_keys_non_existing_columns(destination_config: DestinationTestConfiguration) -> None:
p = destination_config.setup_pipeline("github_3", dev_mode=True)
github_data = github()
# set keys names that do not exist in the data
github_data.load_issues.apply_hints(merge_key=("mA1", "Ma2"), primary_key=("123-x",))
# skip first 45 rows
github_data.load_issues.add_filter(skip_first(45))
info = p.run(github_data, loader_file_format=destination_config.file_format)
assert_load_info(info)
github_1_counts = load_table_counts(p, *[t["name"] for t in p.default_schema.data_tables()])
assert github_1_counts["issues"] == 100 - 45
assert (
p.default_schema.tables["issues"]["columns"]["m_a1"].items()
> {"merge_key": True, "nullable": False}.items()
)

# for non merge destinations we just check that the run passes
if not destination_config.supports_merge:
return

# all the keys are invalid so the merge falls back to append
github_data = github()
github_data.load_issues.apply_hints(merge_key=("mA1", "Ma2"), primary_key=("123-x",))
github_data.load_issues.add_filter(take_first(1))
info = p.run(github_data, loader_file_format=destination_config.file_format)
assert_load_info(info)
github_2_counts = load_table_counts(p, *[t["name"] for t in p.default_schema.data_tables()])
assert github_2_counts["issues"] == 100 - 45 + 1
with p._sql_job_client(p.default_schema) as job_c:
_, storage_cols = job_c.get_storage_table("issues")
storage_cols = normalize_storage_table_cols("issues", storage_cols, p.default_schema)
assert "url" in storage_cols
assert "m_a1" not in storage_cols # unbound columns were not created


@pytest.mark.parametrize(
"destination_config",
destinations_configs(default_sql_configs=True, file_format="parquet"),
Expand Down
2 changes: 1 addition & 1 deletion tests/load/pipeline/test_replace_disposition.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def norm_table_counts(counts: Dict[str, int], *child_tables: str) -> Dict[str, i
offset = 1000

# keep merge key with unknown column to test replace SQL generator
@dlt.resource(name="items", write_disposition="replace", primary_key="id", merge_key="NA")
@dlt.resource(name="items", write_disposition="replace", primary_key="id")
def load_items():
# will produce 3 jobs for the main table with 40 items each
# 6 jobs for the sub_items
Expand Down

0 comments on commit 0d0afa5

Please sign in to comment.