Skip to content

Commit

Permalink
fix issue 360 (#361)
Browse files Browse the repository at this point in the history
  • Loading branch information
Achille authored Oct 4, 2022
1 parent 948ea8c commit 2b6feda
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 12 deletions.
48 changes: 48 additions & 0 deletions parquet_go18_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"bytes"
"fmt"
"log"
"reflect"
"testing"

"github.com/segmentio/parquet-go"
)
Expand Down Expand Up @@ -116,3 +118,49 @@ func ExampleWrite_any() {
// map["FirstName":"Han" "LastName":"Solo"]
// map["FirstName":"R2" "LastName":"D2"]
}

func TestIssue360(t *testing.T) {
type TestType struct {
Key []int
}

schema := parquet.SchemaOf(TestType{})
buffer := parquet.NewGenericBuffer[any](schema)

data := make([]any, 1)
data[0] = TestType{Key: []int{1}}
_, err := buffer.Write(data)
if err != nil {
fmt.Println("Exiting with error: ", err)
return
}

var out bytes.Buffer
writer := parquet.NewGenericWriter[any](&out, schema)

_, err = parquet.CopyRows(writer, buffer.Rows())
if err != nil {
fmt.Println("Exiting with error: ", err)
return
}
writer.Close()

br := bytes.NewReader(out.Bytes())
rows, _ := parquet.Read[any](br, br.Size())

expect := []any{
map[string]any{
"Key": []any{
int64(1),
},
},
}

if !reflect.DeepEqual(rows, expect) {
t.Error("rows mismatch")

for _, row := range rows {
t.Logf("%#v", row)
}
}
}
32 changes: 20 additions & 12 deletions row.go
Original file line number Diff line number Diff line change
Expand Up @@ -578,27 +578,35 @@ func reconstructFuncOfRepeated(columnIndex int16, node Node) (int16, reconstruct
rowLength := nextColumnIndex - columnIndex
return nextColumnIndex, func(value reflect.Value, lvls levels, row Row) (Row, error) {
t := value.Type()
c := value.Cap()
s := value
c := 0
n := 0
if c > 0 {
value.Set(value.Slice(0, c))
} else {
c = 10
value.Set(reflect.MakeSlice(t, c, c))
}
const defaultCapacity = 10

defer func() {
value.Set(value.Slice(0, n))
value.Set(s.Slice(0, n))
}()

if t.Kind() == reflect.Interface {
var v []interface{}
c = defaultCapacity
t = reflect.TypeOf(v)
s = reflect.MakeSlice(t, c, c)
} else if c = s.Cap(); c > 0 {
s = s.Slice(0, c)
} else {
c = defaultCapacity
s = reflect.MakeSlice(t, c, c)
}

return reconstructRepeated(columnIndex, rowLength, lvls, row, func(levels levels, row Row) (Row, error) {
if n == c {
c *= 2
newValue := reflect.MakeSlice(t, c, c)
reflect.Copy(newValue, value)
value.Set(newValue)
newSlice := reflect.MakeSlice(t, c, c)
reflect.Copy(newSlice, s)
s = newSlice
}
row, err := reconstruct(value.Index(n), levels, row)
row, err := reconstruct(s.Index(n), levels, row)
n++
return row, err
})
Expand Down

0 comments on commit 2b6feda

Please sign in to comment.