more on this

dev/get_data.jl

@@ -2,14 +2,19 @@ using Artifacts, ArtifactUtils
 import CodecZlib, Downloads, Tar
 
 # Raw data folder:
-raw_data_dir = "dev/data/fomc-hawkish-dovish-main/data"
+raw_data_root = "dev/data/raw"
+raw_data_dir = joinpath(raw_data_root, "fomc-hawkish-dovish-main/data")
+OVERWRITE = true
 
-if !isdir(raw_data_dir)
+if !isdir(raw_data_dir) || OVERWRITE
+    if isdir(raw_data_root)
+        rm(raw_data_root; recursive=true)
+    end
     tgz = Downloads.download("https://github.com/gtfintechlab/fomc-hawkish-dovish/archive/refs/heads/main.tar.gz")
     open(GzipDecompressorStream, tgz) do io
         Tar.extract(
-            x -> contains(x.path, "fomc-hawkish-dovish-main/data"), 
-            io, "dev/data"
+            x -> contains(x.path, "fomc-hawkish-dovish-main/data") || contains(x.path, "training_data"), 
+            io, "dev/data/raw"
         )
     end
 end

dev/pre_process.jl

@@ -1,8 +1,24 @@
+using Dates
+
 # Get data from the web and save it to a file
 include("get_data.jl")
 
 # Utils:
 include("utils.jl")
 
-# Meetin minutes data:
-include("pre_process_mm.jl")
+# Meeting minutes data:
+include("pre_process_mm.jl")
+
+# Speech data:
+include("pre_process_speech.jl")
+
+# Press conference data:
+include("pre_process_pc.jl")
+
+# Merge:
+df = vcat(df_mm, df_speech, df_pc, cols=:union)
+df.date = Dates.Date.(df.date, "yyyymmdd")
+
+# Save:
+isdir("dev/data/cleaned/") || mkdir("dev/data/cleaned/")
+CSV.write("dev/data/cleaned/all_data.csv", df)