Skip to content

Commit

Permalink
add the map_overlap method to do intrafile filling
Browse files Browse the repository at this point in the history
  • Loading branch information
zain-sohail committed Oct 7, 2023
1 parent 6464926 commit 74df4e5
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions sed/loader/flash/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -737,8 +737,13 @@ def parquet_handler(
# Channels to fill NaN values
channels: List[str] = self.get_channels_by_format(["per_pulse", "per_train"])

# Fill NaN values
dataframe[channels] = dataframe[channels].ffill()
# Define a custom function to forward fill specified columns
def forward_fill_partition(df):
df[channels] = df[channels].ffill()
return df

# Use map_overlap to apply forward_fill_partition
dataframe = dataframe.map_overlap(forward_fill_partition, before=0, after=1)

# Remove the NaNs from per_electron channels
dataframe = dataframe.dropna(
Expand Down

0 comments on commit 74df4e5

Please sign in to comment.