Skip to content

Commit

Permalink
Ensure that schema and Table columns align
Browse files Browse the repository at this point in the history
  • Loading branch information
austinweisgrau committed Jan 19, 2024
1 parent 14721cd commit eb7384b
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 3 deletions.
14 changes: 14 additions & 0 deletions parsons/google/google_bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -775,6 +775,20 @@ def copy(
else:
job_config.schema = self._generate_schema_from_parsons_table(tbl)

# Reorder schema to match table to ensure compatibility
schema = []
for column in tbl.columns:
try:
schema_row = [
i for i in job_config.schema if i.name.lower() == column.lower()
][0]
except IndexError:
raise IndexError(
f"Column found in Table that was not found in schema: {column}"
)
schema.append(schema_row)
job_config.schema = schema

gcs_client = gcs_client or GoogleCloudStorage()
temp_blob_name = f"{uuid.uuid4()}.csv"
temp_blob_uri = gcs_client.upload_table(tbl, tmp_gcs_bucket, temp_blob_name)
Expand Down
4 changes: 1 addition & 3 deletions test/test_databases/test_bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -396,9 +396,7 @@ def test_copy__if_exists_passed_through(self):
bq = self._build_mock_client_for_copying(table_exists=False)
bq.copy_from_gcs = mock.MagicMock()
table_name = "dataset.table"
if_exists = "append"
bq.client.get_table = lambda _: mock.MagicMock()
bq.client.get_table.return_value = None
if_exists = "drop"

# call the method being tested
bq.copy(
Expand Down

0 comments on commit eb7384b

Please sign in to comment.