Skip to content

Commit

Permalink
Run pre commit hooks on code
Browse files Browse the repository at this point in the history
  • Loading branch information
KeilanEvans committed Nov 20, 2024
1 parent df14135 commit 7b81de5
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 28 deletions.
4 changes: 2 additions & 2 deletions rdsa_utils/cdp/helpers/s3_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1164,8 +1164,8 @@ def write_csv(

except Exception as e:
error_message = (
f"Error writing to csv or saving to bucket {bucket_name}, " +
f"filepath {filepath}: {e}"
f"Error writing to csv or saving to bucket {bucket_name}, "
+ f"filepath {filepath}: {e}"
)
logger.error(error_message)
return False
10 changes: 2 additions & 8 deletions rdsa_utils/helpers/pyspark.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,18 +181,12 @@ def wrapper(*args, **kwargs):
varnames = func.__code__.co_varnames
if args:
args = [
(
_convert_to_spark_col(arg)
if varnames[i] not in exclude
else arg
) # noqa: E501
(_convert_to_spark_col(arg) if varnames[i] not in exclude else arg) # noqa: E501
for i, arg in enumerate(args)
]
if kwargs:
kwargs = {
k: (
_convert_to_spark_col(kwarg) if k not in exclude else kwarg
) # noqa: E501
k: (_convert_to_spark_col(kwarg) if k not in exclude else kwarg) # noqa: E501
for k, kwarg in kwargs.items()
}
return func(*args, **kwargs)
Expand Down
24 changes: 6 additions & 18 deletions tests/cdp/helpers/test_s3_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1084,35 +1084,23 @@ def test_write_csv_success(self, s3_client):
data = {"name": ["John"], "age": [30], "city": ["Manchester"]}
df = pd.DataFrame(data)

result = write_csv(s3_client, "test-bucket", df, "test_file.csv")
result = write_csv(s3_client, "test-bucket", df, "test_file.csv")
assert result

def test_write_csv_read_back(self, s3_client):
"""Test that a file wrtitten by write_csv can be read back and returns
the same dataframe as input. Uses kwargs."""
"""Test that a file wrtitten by write_csv can be read back and returns
the same dataframe as input. Uses kwargs."""
data = {"name": ["John"], "age": [30], "city": ["Manchester"]}
df = pd.DataFrame(data)

_ = write_csv(
s3_client,
"test-bucket",
df,
"test_file.csv",
index=False
)
_ = write_csv(s3_client, "test-bucket", df, "test_file.csv", index=False)
result = load_csv(s3_client, "test-bucket", "test_file.csv")
pd.testing.assert_frame_equal(df, result)
pd.testing.assert_frame_equal(df, result)

def test_write_csv_failure(self, s3_client):
"""Test that write_csv returns False if unable to write.
Dictionary data does not have to_csv method."""
data = {"name": ["John"], "age": [30], "city": ["Manchester"]}

result = write_csv(
s3_client,
"test-bucket",
data,
"test_file.csv",
index=False
)
result = write_csv(s3_client, "test-bucket", data, "test_file.csv", index=False)
assert not result

0 comments on commit 7b81de5

Please sign in to comment.