From 951e18d3504e7d44e364eb2d9b54a373345ee454 Mon Sep 17 00:00:00 2001
From: xiaodaigh <zhuojia.dai@gmail.com>
Date: Wed, 13 May 2020 11:13:43 +1000
Subject: [PATCH 1/2] added ability to read missing values; test added

---
 src/reader.jl                                    |   7 ++++++-
 .../parquet-testdata/synthetic_data.parquet      | Bin 0 -> 3996 bytes
 test/test_decode.jl                              |   2 ++
 3 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 test/parquet-compatibility/parquet-testdata/synthetic_data.parquet

diff --git a/src/reader.jl b/src/reader.jl
index 5867669..518d054 100644
--- a/src/reader.jl
+++ b/src/reader.jl
@@ -258,7 +258,12 @@ function read_levels_and_values(io::IO, encs::Tuple, ctype::Int32, num_values::I
 
     #@debug("before reading values bytesavailable in page: $(bytesavailable(io))")
     # read values
-    vals = read_values(io, enc, ctype, num_values)
+    # if there are missing values in the data then
+    # where defn_levels's elements == 1 are present and only
+    # sum(defn_levels) values can be read.
+    # because defn_levels == 0 are where the missing vlaues are
+    nmissing = sum(==(0), defn_levels)
+    vals = read_values(io, enc, ctype, num_values - nmissing)
 
     vals, defn_levels, repn_levels
 end
diff --git a/test/parquet-compatibility/parquet-testdata/synthetic_data.parquet b/test/parquet-compatibility/parquet-testdata/synthetic_data.parquet
new file mode 100644
index 0000000000000000000000000000000000000000..0ebcb902069e742287b6efbab5eb158b3e8c7c86
GIT binary patch
literal 3996
zcmb_fOKcle6dl`%YtmG$s*Z7r7m)>4fVB7%r*S0|p1&E_aT}+xJ?0bU7kk{9vGFIi
zQ!24w!2&1?eku|hR24x50;(<wQdj^|QH4Mt#0CVVd_-9wU;zRI_q`b>cH$yF8D(bP
zyZ60&&%NiqJI)c5_R<6Ny|kF5hv|Xc6y<s23S)Zs+ds~&p8xHIlV87lb@lY)r=LI{
zelq>ujVq5tre2)hdExGzUq1E2>Yrcjxb<W9yH%Q^9kg?tc7h=*ZFM?qogEHaXRUqL
zSRW}xQN!aFcn(IL7Ry8LnBC8so%GO1V1TlYI4rG2FKxCv?T?!6Pnz8incTL2XB+U`
zK}V9bjRZD&{|6^0E)L%F{`B^%(US)r`+RjFdj6i*KD<0_Kjk|2T=Nwih@IZs9Y|+|
z!`9hj-!o?0EQJuZkC8ykQ)bH*%Kg${PaHlH$M+`t&nEY&0rv$91hU!6HEnNX=;~SD
z?``4R)wAzjw7h!dv+1+fzy51>&o|SDZhk-i)0sb3e;jz`P1~8bSDhE0{`A5PpZ8&?
zv%_KQ?7<9emaa2&dAH8cWm`{V{w{_tZF673^lvtEP2&*RQ96^PhX_TtL<YbiuQVhp
zyxWqZl$)mANukYYIJs3x9E4Y)DoNVLqMO_`X4N-12CxBGXOz}XD*&G~yFc3Qq=${t
zvI=FT+^7=l@0si$ncP1PxgQvE_OSv8C+MjpR5m^xd&O;FS(6&|DyLwug>qTM*mzFd
zSU!`x$F><O9Tw2TDx*lIhRv1-P3~WItRor}6s}eyfX<ri=S=RWcevjgcJ{GBn@5xY
z<*?R-qNH;45JeB&WG07YZRA@U`6+rBv_`x)QAsDDAR^tNA+bA0oFjK_?djX%GvMCZ
z!uzk~@E=LDKYyLwNsWv<tQAfZxtimO^|(vcM5(n1rPxrln(N@`oyT3rKn5dMmD386
zYkbtDajFCYt)VC|tJS$`9i|>KE>~daogib0j8EJR6W!n<u`U^Qd(AA#k|Hkl%5+?d
z^>zjOb2T(sD2qCThFWQ_Daw3F66zS5D%ClWt8=jYxJ%MtxvUlN-i4#FeN(OzU0`}q
zxS+9P13fT0!VNf<lrmSJoFJ*_!O^AY0)q2&VjS=o!GTp@KUYM!<*xkwToK_GyVBd&
zW^e2A4G#Yuotv^qno}QlxMyMU|Mx};w<>Y<a+TDRBJ)+Q+Q#KnX>-+TxrzNOOVyfO
z*09vG-y7KPbphEO?>$OiV87pWf^rO_boy}ipx!JU-w)r0uc<;5Av6a!z78QYfkFio
zL#Tkl1tJTfX1BhHg|NraCz=s`4o_l*=oti?LI}Z|7=bt01Wy+3P?OCDj<LB}gyM6s
zKXiaml}0|EnaKH4%lu5H&TF&sVL8N0?L?J>wM?m8V6;S$S4D-9n*~M<6#47|Uege>
zty(gSWWk><3fdgVP}7(%3t?uGi9|idT+!#TU~V4UFnlS&3a_RWDV|xuezN|W%&W0_
zejfJ5171GftTEw)%4IXPd@R^tvt9+S<F!nU4}%>o1h~MrH|UXaA);fnOoPj&mh<ri
z8T*E{bzw$X5@(bqu^Ub-vuZ_Q{d2lLoAI#ObWw~)aZHepOoR0Wk3sAjiCxJoq-D<p
zqj`I=ujgqE7o!!IX6Ko{n7eca2dQpMDjyFP#R%&$a_r@Ep+L@`4hRqj<W<hAtz|Yh
z=Szi&{dzu^F6QGg54)DDRxlaH_&fX{S&B`yb$j_rKHEB$^Yzj7%b}XD@xGWRRWfq4
zIfonYdKEDqgStu=*Z7(7)R>trKEsKL+{!G;d0tf%KAv7$<GM>H#A8s8v6a7x9qP{8
zlrmkMZ^YZnDXG&Xs5_|Xb*^v?#)Do#z6M{t`hJ{wc~vR#zI5kYY~qafCGN{-QwFF1
zC`Tn8Vi(%i#8hK*sTIO$Ki>Xd?dy5U0B_+$Nl-IzCW39uC*-ZhWfN^Smk5wEp@jl4
zZu5SD@F}X9w%}8ic^S@am}w?UQ7_yw&Ux1v_2qMk6)qcW5E~`bm@O?dX4{D%?okLW
zz#Sp|s)VO;{D5W#7xXiPVk@|5!XO%hCJ9%kACd_|sS?%`73*si1wZsS6prFn3xiN7
zP5L3}l*FFC4f^SvZeQ<@z-Mb&%mIvKq~kBr1qJtP{Yo4#^sp8~nSrz4jIbSls73r?
z5Qcp`io^|l7$K|*!I@xiQ-|OSxc|=>adkk!ny8+X5P`n#AMY;UKF{K}eiab@CLlQX
zEkklU<$zC9^}dEw-zQWmqsI4X)Vm)(T>Iuz=*axc;iDsW!B6t;KkAF^6m<mtFaHZN
CBgc#Y

literal 0
HcmV?d00001

diff --git a/test/test_decode.jl b/test/test_decode.jl
index f8d87e5..bda7089 100644
--- a/test/test_decode.jl
+++ b/test/test_decode.jl
@@ -54,3 +54,5 @@ function test_decode_all_pages()
 end
 
 test_decode_all_pages()
+
+test_decode("parquet-testdata/synthetic_data.parquet")

From aba9ec3c3650498df631208c92a0261f23a3623c Mon Sep 17 00:00:00 2001
From: xiaodaigh <zhuojia.dai@gmail.com>
Date: Wed, 13 May 2020 14:12:46 +1000
Subject: [PATCH 2/2] moved test data synthetic data as required.

---
 .../synthetic_data.parquet                          | Bin
 test/test_decode.jl                                 |   2 +-
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename test/{parquet-compatibility/parquet-testdata => missingvalues}/synthetic_data.parquet (100%)

diff --git a/test/parquet-compatibility/parquet-testdata/synthetic_data.parquet b/test/missingvalues/synthetic_data.parquet
similarity index 100%
rename from test/parquet-compatibility/parquet-testdata/synthetic_data.parquet
rename to test/missingvalues/synthetic_data.parquet
diff --git a/test/test_decode.jl b/test/test_decode.jl
index bda7089..1a34895 100644
--- a/test/test_decode.jl
+++ b/test/test_decode.jl
@@ -55,4 +55,4 @@ end
 
 test_decode_all_pages()
 
-test_decode("parquet-testdata/synthetic_data.parquet")
+test_decode("synthetic_data.parquet", "missingvalues")