From 5ef508e9d9fbc5fdff32deebcdd8612ddab97786 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 22 Oct 2021 14:44:47 +0000 Subject: [PATCH 1/3] Fixed error with csv files. --- modules/csv/csv.go | 24 ++++++++---------------- routers/web/repo/compare.go | 23 +++++++++++++++-------- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 30698830a478d..c8baf7054a690 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -27,9 +27,10 @@ func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { } // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader. +// Reads at most 10k bytes. func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { - var data = make([]byte, 1e4) - size, err := rd.Read(data) + var buf = make([]byte, 1e4) + n, err := rd.Read(buf) if err != nil { if err == io.EOF { return CreateReader(bytes.NewReader([]byte{}), rune(',')), nil @@ -37,25 +38,16 @@ func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { return nil, err } - delimiter := guessDelimiter(data[:size]) - - var newInput io.Reader - if size < 1e4 { - newInput = bytes.NewReader(data[:size]) - } else { - newInput = io.MultiReader(bytes.NewReader(data), rd) - } - - return CreateReader(newInput, delimiter), nil + return CreateReader( + io.MultiReader(bytes.NewReader(buf[:n]), rd), + guessDelimiter(buf[:n]), + ), nil } // guessDelimiter scores the input CSV data against delimiters, and returns the best match. -// Reads at most 10k bytes & 10 lines. func guessDelimiter(data []byte) rune { maxLines := 10 - maxBytes := util.Min(len(data), 1e4) - text := string(data[:maxBytes]) - text = quoteRegexp.ReplaceAllLiteralString(text, "") + text := quoteRegexp.ReplaceAllLiteralString(string(data), "") lines := strings.SplitN(text, "\n", maxLines+1) lines = lines[:util.Min(maxLines, len(lines))] diff --git a/routers/web/repo/compare.go b/routers/web/repo/compare.go index 33b95838c7bea..9557b79bdf938 100644 --- a/routers/web/repo/compare.go +++ b/routers/web/repo/compare.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "html" + "io" "net/http" "path" "path/filepath" @@ -105,30 +106,36 @@ func setCsvCompareContext(ctx *context.Context) { errTooLarge := errors.New(ctx.Locale.Tr("repo.error.csv.too_large")) - csvReaderFromCommit := func(c *git.Commit) (*csv.Reader, error) { + csvReaderFromCommit := func(c *git.Commit) (*csv.Reader, io.Closer, error) { blob, err := c.GetBlobByPath(diffFile.Name) if err != nil { - return nil, err + return nil, nil, err } if setting.UI.CSV.MaxFileSize != 0 && setting.UI.CSV.MaxFileSize < blob.Size() { - return nil, errTooLarge + return nil, nil, errTooLarge } reader, err := blob.DataAsync() if err != nil { - return nil, err + return nil, nil, err } - defer reader.Close() - return csv_module.CreateReaderAndGuessDelimiter(charset.ToUTF8WithFallbackReader(reader)) + csvReader, err := csv_module.CreateReaderAndGuessDelimiter(charset.ToUTF8WithFallbackReader(reader)) + return csvReader, reader, err } - baseReader, err := csvReaderFromCommit(baseCommit) + baseReader, baseBlobCloser, err := csvReaderFromCommit(baseCommit) + if baseBlobCloser != nil { + defer baseBlobCloser.Close() + } if err == errTooLarge { return CsvDiffResult{nil, err.Error()} } - headReader, err := csvReaderFromCommit(headCommit) + headReader, headBlobCloser, err := csvReaderFromCommit(headCommit) + if headBlobCloser != nil { + defer headBlobCloser.Close() + } if err == errTooLarge { return CsvDiffResult{nil, err.Error()} } From 989cca733a54aca0e4b6d353f70419a2e3e72f9c Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 25 Oct 2021 00:18:34 +0200 Subject: [PATCH 2/3] use data --- modules/csv/csv.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 3e9f42888fdbb..e79b01a0402c4 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -36,8 +36,8 @@ func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { } return CreateReader( - io.MultiReader(bytes.NewReader(buf[:n]), rd), - guessDelimiter(buf[:n]), + io.MultiReader(bytes.NewReader(data[:n]), rd), + guessDelimiter(data[:n]), ), nil } From f827bae332124be267ac492b4ace50a120003742 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 25 Oct 2021 00:20:40 +0200 Subject: [PATCH 3/3] use size --- modules/csv/csv.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index e79b01a0402c4..c239c7f647ace 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -36,8 +36,8 @@ func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { } return CreateReader( - io.MultiReader(bytes.NewReader(data[:n]), rd), - guessDelimiter(data[:n]), + io.MultiReader(bytes.NewReader(data[:size]), rd), + guessDelimiter(data[:size]), ), nil }