Merge pull request #43 from aaowens/use2019

Change to 2019 files, use official PSID codebook

.travis.yml

@@ -4,6 +4,5 @@ os:
     - osx
     - windows
 julia:
-    - 1.4
-    - 1.5
+    - 1.6
 codecov: true

README.md

@@ -15,15 +15,15 @@ An example workflow can be found on my blog post [here](https://aaowens.github.i
 
 To add this package, use
 ```
-(@v1.4) pkg> add PSID
+(@v1.6) pkg> add PSID
 ```
 
 Next, download the PSID data files yourself. The package can't automatically fetch them because the PSID requires you to register for a free account before using the data.
 
 The list of data files required to be in the current directory can be found [here](https://github.com/aaowens/PSID.jl/blob/master/src/allfiles_hash.json). These files are
 
-1. The PSID codebook in XML format. You can download this from me here https://drive.google.com/open?id=1nz1UaVGcj0ur2Bp3ev7a8agJbj0A5JTF . In the future there will be a way to download this from the PSID directly.
-2. The zipped PSID family files and cross-year individual file, which can be downloaded here https://simba.isr.umich.edu/Zips/ZipMain.aspx. Do not extract the files--leave them zipped. You need to download every family file from 1968 to 2017, and you also need to download the cross-year individual file.
+1. The PSID codebook in XML format. You can download this from the PSID here https://simba.isr.umich.edu/downloads/PSIDCodebook.zip .
+2. The zipped PSID family files and cross-year individual file, which can be downloaded here https://simba.isr.umich.edu/Zips/ZipMain.aspx. Do not extract the files--leave them zipped. You need to download every family file from 1968 to 2019, and you also need to download the cross-year individual file.
 3. The XLSX cross-year index for the variables, which can be downloaded here https://psidonline.isr.umich.edu/help/xyr/psid.xlsx.
 
 After acquiring the data, run

src/allfiles_hash.json

@@ -1,6 +1,6 @@
 {
-  "J265684_codebook.xml": "6d9a40ef8c61aa359a31aa4d04746b31511893a05e81337aa7e7a365c8452f0a",
-  "psid.xlsx": "9f277e239d8483a3c852b7214bee6ac7d54cb101b27b4f95f75058e186b435d2",
+  "PSIDCodebook.xml": "0aa45e5024297f1b319b873f475006598ff6f275dd586064eb7eb7b753d67eb6",
+  "psid.xlsx": "4f3755c40d4057ccc0443077cf62f77ecc62c35e4bec018c34a91a29a3d98961",
   "fam1968.zip": "38292539d020824be3ca3908c04093edf8d3ccef8dc44cb81147a2315e94cd00",
   "fam1969.zip": "7942f14d8c0c3f42c05efa3fd6b90b011721e280f198fe6b78a417bb28dc9335",
   "fam1970.zip": "07bf03d9aa9ca9258aff3546abf60e66447c02031819d5a552f4769ddb8bb90f",
@@ -41,5 +41,6 @@
   "fam2013er.zip": "43a527b834dc31b753881d3ab03fe1a4c4f1dde7eb5aa2d77a7c3bb79095d15c",
   "fam2015er.zip": "726236d3f9d25e804d2605eff6a6a11f322999530d27a2528d0d01cf31af6066",
   "fam2017er.zip": "5ade1a3f42ed84c892fe8ff16365b85b0dc84ac66f4d454e291af12008e9b35d",
-  "ind2017er.zip": "7ea5837017603841afeb0d4d0365745d1a592e5c7a77021d4e1d617b7aed486c"
+  "fam2019er.zip": "a5a19dbbff36ac9dab7f5ddc46b35ace0ae64e924b51e7e66507c1f8850a8bb7",
+  "ind2019er.zip": "583e541f2f8d8c9ea12e2f53e8f7594f525d95ad929f5ce804d030e1a593991d"
 }

src/construct_alldata.jl

@@ -19,7 +19,7 @@ function construct_alldata(famdatas, inddata; codemissings = true)
     #readme = procvar
     readme_ind = JSON3.read(read("output/ind_output.json", String), Vector{VarInfo5})
 
-    years = [collect(1968:1997); collect(1999:2:2017)]
+    years = [collect(1968:1997); collect(1999:2:2019)]
     newdatas_ind = [DataFrame() for x in famdatas]
     # Go year by year
     for y_nx in eachindex(years)

src/process_codebook.jl

@@ -3,7 +3,7 @@ AbstractTrees.printnode(io::IO, x::AbstractXMLNode) = print(io, name(x));
 
 #Processes XML codebook tree into a usable JSON table
 function process_codebook()
-    xdoc = parse_file("J265684_codebook.xml");
+    xdoc = parse_file("PSIDCodebook.xml");
     r = root(xdoc);
     t = Tree(r)
     # Travel down the tree until we reach the list of variables

src/unzip_data.jl

@@ -72,9 +72,9 @@ function readPSID(zipname)
 end
 
 function unzip_data()
-    years = [collect(1968:1997); collect(1999:2:2017)]
+    years = [collect(1968:1997); collect(1999:2:2019)]
     filenames = [year <= 1993 ? "fam$year" : "fam$(year)er" for year in years]
     datas =  SortedDict(year => readPSID(filename) for (year, filename) in zip(years, filenames))
-    inddata = readPSID("ind2017er")
+    inddata = readPSID("ind2019er")
     (famdatas = datas, inddata = inddata)
 end

test/runtests.jl

@@ -16,8 +16,8 @@ end
 
 if skipdata
     Base.download("https://raw.githubusercontent.com/aaowens/PSID.jl/master/examples/user_input.json", "user_input.json")
-    Base.download("https://drive.google.com/uc?authuser=0&id=1nz1UaVGcj0ur2Bp3ev7a8agJbj0A5JTF&export=download", "J265684_codebook.zip")
-    run(DataDeps.unpack_cmd("J265684_codebook.zip", "$(pwd())", ".zip", ""))
+    Base.download("https://simba.isr.umich.edu/downloads/PSIDCodebook.zip", "PSIDCodebook.zip")
+    run(DataDeps.unpack_cmd("PSIDCodebook.zip", "$(pwd())", ".zip", ""))
     Base.download("https://psidonline.isr.umich.edu/help/xyr/psid.xlsx", "psid.xlsx")
     userinput_json = "user_input.json"
     isfile(userinput_json) || error("$userinput_json not found in current directory")