read in json from hdfs (#27)

* initial commit for reading in json from hdfs * code is in a function and pydoop added to requirements.txt * moved read into main.py * comment out r and d * remove old file
ONSdigital · Jun 13, 2024 · c5d1f5b · c5d1f5b
1 parent 9a040a1
commit c5d1f5b
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 0 deletions.
diff --git a/main.py b/main.py
@@ -0,0 +1,13 @@
+import pandas as pd
+
+from src.utils.hdfs_mods import hdfs_load_json as read_json
+
+# TODO: read from config
+folder_path = "/dapsen/workspace_zone/mbs-results/"
+file_name = "snapshot-202212-002-2156d36b-e61f-42f1-a0f1-61d1f8568b8e.json"
+file_path = folder_path + file_name
+
+snapshot = read_json(file_path)
+
+contributors = pd.DataFrame(snapshot["contributors"])
+responses = pd.DataFrame(snapshot["responses"])
diff --git a/requirements.txt b/requirements.txt
@@ -9,6 +9,7 @@ black
 isort
 nbstripout
 nbqa
+#research_and_development==1.0.0
 pre_commit_hooks
 flake8
 pandas==1.1.5