remove pgloader, load directly into postgres using ./reparse.sh

README.md

@@ -113,13 +113,12 @@ building and tax period the data applies to.
 
 ### To import the CSV into postgres
 
-*In progress*
+You should have [docker4data](http://dockerfordata.com) installed and set up on
+your system.
 
-There are a few complicated dependencies here, including
-[pgloader](http://pgloader.io), and a few external tables (PLUTO and the DHCR
-stabilization building list history.)
+    ./reparse.sh
 
-    ./import.sh
+This will directly parse the `data` folder into docker4data's postgres.
 
 ## Data Usage
 

import.sh

@@ -1,14 +1,9 @@
 #!/bin/bash -e
 
-# createdb stabilization 2>/dev/null || :
-
 export PGPASSWORD=docker4data
 export PGUSER=postgres
 export PGHOST=localhost
 export PGPORT=54321
 export PGDATABASE=postgres
 
-# need to have pgloader installed
-pgloader pgloader.load
-
 psql -f cross-tab-rs-counts.sql

parse.py

@@ -30,6 +30,8 @@
     "apts"
 ]
 
+ROW_BUFFER = 10000
+
 BILL_PDF, STATEMENT_PDF, STATEMENT_HTML, NOPV_PDF, NOPV_HTML = (
     'Quarterly Property Tax Bill.pdf', 'Quarterly Statement of Account.pdf',
     'Quarterly Statement of Account.html', 'Notice of Property Value.pdf',
@@ -486,6 +488,7 @@ def main(root): #pylint: disable=too-many-locals,too-many-branches,too-many-stat
     """
     writer = csv.DictWriter(sys.stdout, HEADERS)
     writer.writeheader()
+    rows_to_write = []
     for path, _, files in os.walk(root):
         bbl_json = []
         for filename in sorted(files):
@@ -514,12 +517,10 @@ def main(root): #pylint: disable=too-many-locals,too-many-branches,too-many-stat
                         file_data = handle.read()
                     activity_through = parsedate(filename.split(' - ')[0])
                     for data in handler(file_data):
-                        base = {
-                            'bbl': ''.join(bbl_array),
-                            'activityThrough': activity_through
-                        }
-                        base.update(data)
-                        writer.writerow(base)
+                        data['bbl'] = ''.join(bbl_array)
+                        data['activityThrough'] = activity_through
+                        #writer.writerow(base)
+                        rows_to_write.append(data)
                         bbl_json.append(data)
 
             except Exception as err:  # pylint: disable=broad-except
@@ -528,5 +529,11 @@ def main(root): #pylint: disable=too-many-locals,too-many-branches,too-many-stat
         with open(os.path.join(path, 'data.json'), 'w') as json_outfile:
             json.dump(bbl_json, json_outfile)
 
+        if len(rows_to_write) >= ROW_BUFFER:
+            writer.writerows(rows_to_write)
+            rows_to_write = []
+    writer.writerows(rows_to_write)
+
+
 if __name__ == '__main__':
     main(sys.argv[1])

reparse.sh

@@ -1,4 +1,50 @@
 #!/bin/bash
 
 source .env/bin/activate
-time python parse.py data/ >data/rawdata.csv 2>data/rawdata.log &
+
+export PGPASSWORD=docker4data
+export PGUSER=postgres
+export PGHOST=localhost
+export PGPORT=54321
+export PGDATABASE=postgres
+
+psql -c 'drop table if exists rawdata cascade;'
+psql -c 'create table rawdata (
+         bbl bigint,
+         activityThrough DATE,
+         section TEXT,
+         key TEXT,
+         dueDate DATE,
+         activityDate DATE,
+         value TEXT,
+         meta TEXT,
+         apts TEXT
+        );'
+psql -c 'drop table if exists rgb cascade;'
+psql -c 'create table rgb (
+         source VARCHAR,
+         borough SMALLINT,
+         year INT,
+         add_421a INT,
+         add_421g INT,
+         add_420c INT,
+         add_j51 INT,
+         add_ML_buyout INT,
+         add_loft INT,
+         add_former_control REAL,
+         sub_high_rent_income INT,
+         sub_high_rent_vacancy INT,
+         sub_coop_condo_conversion INT,
+         sub_421a_expiration INT,
+         sub_j51_expiration INT,
+         sub_substantial_rehab INT,
+         sub_commercial_prof_conversion INT,
+         sub_other INT,
+         total_sub INT,
+         total_add REAL,
+         inflated VARCHAR,
+         net REAL
+        );'
+time cat data/rgb.csv | psql -c "COPY rgb FROM stdin WITH CSV HEADER NULL '' QUOTE'\"';"
+
+time python parse.py data/ 2>data/rawdata.log | psql -c "COPY rawdata FROM stdin WITH CSV HEADER NULL '' QUOTE '\"';"