Skip to content

Commit

Permalink
fixed lint
Browse files Browse the repository at this point in the history
  • Loading branch information
HarishC727 committed Nov 25, 2024
1 parent fa8afef commit 7475700
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 13 deletions.
11 changes: 7 additions & 4 deletions scripts/oecd/regional_demography/deaths/preprocess_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
_MODULE_DIR = os.path.dirname(os.path.abspath(__file__))
flags.DEFINE_string('mode', '', 'Options: download or process')


logging.basicConfig(level=logging.INFO)

_MODULE_DIR = os.path.dirname(os.path.abspath(__file__))
Expand Down Expand Up @@ -149,7 +148,7 @@ def process_data(df, output_file_path):
for col in columns_to_drop:
if col in df_cleaned.columns:
df_cleaned.drop(col, axis=1, inplace=True)
logging.info("Writing output to %s",output_file_path)
logging.info("Writing output to %s", output_file_path)
df_cleaned.to_csv(output_file_path, index=False, quoting=csv.QUOTE_NONE)

return df_cleaned
Expand Down Expand Up @@ -184,13 +183,17 @@ def main(_):
filename = os.path.join(_MODULE_DIR, "REGION_DEMOGR_death_5Y.csv")

if mode == "" or mode == "download":
download_data_to_file_and_df(url, filename, is_download_required=True,csv_filepath=None)
download_data_to_file_and_df(url,
filename,
is_download_required=True,
csv_filepath=None)
if mode == "" or mode == "process":
df = pd.read_csv(filename)
output_file_path = os.path.join(_MODULE_DIR, "OECD_deaths_cleaned.csv")
df_cleaned = process_data(df, output_file_path)
filepath = os.path.join(_MODULE_DIR, "OECD_deaths.tmcf")
generate_tmcf(df_cleaned, filepath)



if __name__ == "__main__":
app.run(main)
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ def process_data(df, output_file_path):

df_cleaned.rename(columns=VAR_to_statsvars, inplace=True)
df_cleaned.drop(columns=["REG_ID"], inplace=True)
logging.info("Writing output to %s",output_file_path)
logging.info("Writing output to %s", output_file_path)
df_cleaned.to_csv(output_file_path, index=False, quoting=csv.QUOTE_NONE)
return df_cleaned

Expand Down Expand Up @@ -121,20 +121,25 @@ def generate_tmcf(df_cleaned, filepath):
'stat_var': stat_vars[i]
}))


def main(_):
mode = _FLAGS.mode
url = "https://sdmx.oecd.org/public/rest/data/OECD.CFE.EDS,DSD_REG_DEMO@DF_LIFE_EXP,2.0/all?dimensionAtObservation=AllDimensions&format=csvfilewithlabels"
filename = os.path.join(_MODULE_DIR, "REGION_DEMOGR_life_expectancy.csv")

if mode == "" or mode == "download":
download_data_to_file_and_df(url, filename, is_download_required=True,csv_filepath=None)
download_data_to_file_and_df(url,
filename,
is_download_required=True,
csv_filepath=None)
if mode == "" or mode == "process":
df = pd.read_csv(filename)
output_file_path = os.path.join(_MODULE_DIR,
"OECD_life_expectancy_cleaned.csv")
"OECD_life_expectancy_cleaned.csv")
df_cleaned = process_data(df, output_file_path)
filepath = os.path.join(_MODULE_DIR, "OECD_life_expectancy.tmcf")
generate_tmcf(df_cleaned, filepath)


if __name__ == "__main__":
app.run(main)
11 changes: 8 additions & 3 deletions scripts/oecd/regional_demography/pop_density/preprocess_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def process_data(df, output_file_path):
}

df_cleaned.rename(columns=VAR_to_statsvars, inplace=True)
logging.info("Writing output to %s",output_file_path)
logging.info("Writing output to %s", output_file_path)
df_cleaned.to_csv(output_file_path, index=False, quoting=csv.QUOTE_NONE)


Expand All @@ -82,11 +82,16 @@ def main(_):
filename = os.path.join(_MODULE_DIR, "REGION_DEMOGR_pop_density.csv")

if mode == "" or mode == "download":
download_data_to_file_and_df(url, filename, is_download_required=True,csv_filepath=None)
download_data_to_file_and_df(url,
filename,
is_download_required=True,
csv_filepath=None)
if mode == "" or mode == "process":
df = pd.read_csv(filename)
output_file_path = os.path.join(_MODULE_DIR, "OECD_pop_density_cleaned.csv")
output_file_path = os.path.join(_MODULE_DIR,
"OECD_pop_density_cleaned.csv")
process_data(df, output_file_path)


if __name__ == "__main__":
app.run(main)
10 changes: 7 additions & 3 deletions scripts/oecd/regional_demography/population/preprocess_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ def process_data(df, output_file_path):
if col not in csv_columns:
drop_cols.append(col)
df_cleaned.drop(columns=drop_cols, axis=0, inplace=True)
logging.info("Writing output to %s",output_file_path)
logging.info("Writing output to %s", output_file_path)
df_cleaned.to_csv(output_file_path, index=False, quoting=csv.QUOTE_NONE)
return df_cleaned

Expand Down Expand Up @@ -172,13 +172,17 @@ def main(_):
filename = os.path.join(_MODULE_DIR, "REGION_DEMOGR_population.csv")

if mode == "" or mode == "download":
download_data_to_file_and_df(url, filename, is_download_required=True,csv_filepath=None)
download_data_to_file_and_df(url,
filename,
is_download_required=True,
csv_filepath=None)
if mode == "" or mode == "process":
df = pd.read_csv(filename)
csv_filepath = os.path.join(_MODULE_DIR, "OECD_population_cleaned.csv")
df_cleaned = process_data(df, csv_filepath)
tmcf_filepath = os.path.join(_MODULE_DIR, "OECD_population.tmcf")
generate_tmcf(df_cleaned, tmcf_filepath)

if __name__=="__main__":

if __name__ == "__main__":
app.run(main)

0 comments on commit 7475700

Please sign in to comment.