Skip to content

Commit

Permalink
Add scan time support in soda-spark (#111)
Browse files Browse the repository at this point in the history
* Add scan time support in soda-spark

Co-authored-by: Vijay Kiran <[email protected]>
  • Loading branch information
shannywu and vijaykiran authored May 11, 2022
1 parent 45be6f9 commit 5d340e4
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 1 deletion.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
- Add `time` support to `execute`

## [0.3.1] - 2021-12-27

- Add example to README about returning spark data frames as scan result
Expand Down
13 changes: 12 additions & 1 deletion src/sodaspark/scan.py
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,7 @@ def create_scan(
variables: dict | None = None,
warehouse_name: str = "sodaspark",
soda_server_client: SodaServerClient | None = None,
time: str | None = None,
) -> Scan:
"""
Create a scan object.
Expand All @@ -265,19 +266,25 @@ def create_scan(
variables: variables to be substituted in scan yml
soda_server_client : Optional[SodaServerClient] (default : None)
A soda server client.
time: Optional[str] (default: None)
Timestamp date in ISO8601 format. If None, use datatime.now() in ISO8601 format.
Returns
-------
out : Scan
The scan.
"""

time = time or dt.datetime.now(tz=dt.timezone.utc).isoformat(
timespec="seconds"
)
warehouse = create_warehouse(warehouse_name)
scan = Scan(
warehouse=warehouse,
scan_yml=scan_yml,
soda_server_client=soda_server_client,
variables=variables,
time=dt.datetime.now(tz=dt.timezone.utc).isoformat(timespec="seconds"),
time=time,
)
return scan

Expand Down Expand Up @@ -422,6 +429,7 @@ def execute(
warehouse_name: str = "sodaspark",
soda_server_client: SodaServerClient | None = None,
as_frames: bool | None = False,
time: str | None = None,
) -> ScanResult:
"""
Execute a scan on a data frame.
Expand All @@ -438,6 +446,8 @@ def execute(
A soda server client.
as_frames : bool (default : False)
Flag to return results in Dataframe
time: str (default : None)
Timestamp date in ISO8601 format at the start of a scan
Returns
-------
Expand All @@ -451,6 +461,7 @@ def execute(
scan_yml,
variables=variables,
soda_server_client=soda_server_client,
time=time,
warehouse_name=warehouse_name,
)
scan.execute()
Expand Down

0 comments on commit 5d340e4

Please sign in to comment.