From 1813517c2358976b679a42238208c3abea1afbe7 Mon Sep 17 00:00:00 2001 From: Javad Date: Tue, 9 Jul 2024 12:39:45 +0330 Subject: [PATCH 1/9] feat: add file downloader with realtime stats - stop/pause/resume operation in downloading - realtime stats download - error signal and known errors - file details --- .gitignore | 2 + util/downloader/downloader.go | 217 +++++++++++++++++++++++++++++ util/downloader/downloader_test.go | 126 +++++++++++++++++ 3 files changed, 345 insertions(+) create mode 100644 util/downloader/downloader.go create mode 100644 util/downloader/downloader_test.go diff --git a/.gitignore b/.gitignore index 2a955a371..a2133b490 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ todo # VIM artifacts .swp .*.sw* + +util/downloader/testdata/ diff --git a/util/downloader/downloader.go b/util/downloader/downloader.go new file mode 100644 index 000000000..911e22f5a --- /dev/null +++ b/util/downloader/downloader.go @@ -0,0 +1,217 @@ +package downloader + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sync" +) + +var ( + ErrHeaderRequest = errors.New("request header error") + ErrSHA256Mismatch = errors.New("sha256 mismatch") + ErrCreateDir = errors.New("create dir error") + ErrOpenFile = errors.New("open file error") + ErrInvalidFilePath = errors.New("file path is a directory, not a file") + ErrGetFileInfo = errors.New("get file info error") + ErrCopyExistsFileData = errors.New("error copying existing file data") + ErrDoRequest = errors.New("error doing request") + ErrFileWriting = errors.New("error writing file") + ErrNewRequest = errors.New("error creating request") + ErrOpenFileExists = errors.New("error opening file exists") +) + +type Downloader struct { + url string + filePath string + sha256Sum string + fileType string + fileName string + pause chan bool + resume chan bool + stop chan bool + statsCh chan Stats + errCh chan error + wg sync.WaitGroup +} + +type Stats struct { + Downloaded int64 + TotalSize int64 + Percent float64 + Completed bool +} + +func NewDownloader(url, filePath, sha256Sum string) *Downloader { + return &Downloader{ + url: url, + filePath: filePath, + sha256Sum: sha256Sum, + pause: make(chan bool), + resume: make(chan bool), + stop: make(chan bool), + statsCh: make(chan Stats), + errCh: make(chan error, 1), + } +} + +func (d *Downloader) Start() { + d.wg.Add(1) + go d.download() +} + +func (d *Downloader) Pause() { + d.pause <- true +} + +func (d *Downloader) Resume() { + d.resume <- true +} + +func (d *Downloader) Stop() { + d.stop <- true + d.wg.Wait() + close(d.statsCh) + close(d.errCh) +} + +func (d *Downloader) Stats() <-chan Stats { + return d.statsCh +} + +func (d *Downloader) FileType() string { + return d.fileType +} + +func (d *Downloader) FileName() string { + return d.fileName +} + +func (d *Downloader) Errors() <-chan error { + return d.errCh +} + +func (d *Downloader) download() { + defer d.wg.Done() + + resp, err := http.Head(d.url) + if err != nil { + d.handleError(ErrHeaderRequest) + return + } + + stats := Stats{ + TotalSize: resp.ContentLength, + } + + d.fileType = resp.Header.Get("Content-Type") + d.fileName = filepath.Base(d.filePath) + + dir := filepath.Dir(d.filePath) + if err := os.MkdirAll(dir, 0755); err != nil { + d.handleError(ErrCreateDir) + return + } + + fileInfo, err := os.Stat(d.filePath) + if err == nil && fileInfo.IsDir() { + d.handleError(ErrInvalidFilePath) + return + } + + out, err := os.OpenFile(d.filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + d.handleError(ErrOpenFile) + return + } + defer out.Close() + + fileInfo, err = out.Stat() + if err != nil { + d.handleError(ErrGetFileInfo) + return + } + stats.Downloaded = fileInfo.Size() + + req, err := http.NewRequest("GET", d.url, nil) + if err != nil { + d.handleError(ErrNewRequest) + return + } + if stats.Downloaded > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", stats.Downloaded)) + } + + client := &http.Client{} + resp, err = client.Do(req) + if err != nil { + d.handleError(ErrDoRequest) + return + } + defer resp.Body.Close() + + buffer := make([]byte, 32*1024) + hasher := sha256.New() + + // Update the hasher with the already downloaded part + if stats.Downloaded > 0 { + existingFile, err := os.Open(d.filePath) + if err != nil { + d.handleError(ErrOpenFileExists) + return + } + defer existingFile.Close() + if _, err := io.CopyN(hasher, existingFile, stats.Downloaded); err != nil { + d.handleError(ErrCopyExistsFileData) + return + } + } + + for { + select { + case <-d.pause: + <-d.resume + case <-d.stop: + return + default: + n, err := resp.Body.Read(buffer) + if n > 0 { + if _, err := out.Write(buffer[:n]); err != nil { + d.handleError(ErrFileWriting) + return + } + hasher.Write(buffer[:n]) + stats.Downloaded += int64(n) + stats.Percent = float64(stats.Downloaded) / float64(stats.TotalSize) * 100 + d.statsCh <- stats + } + if err != nil { + if err == io.EOF { + stats.Completed = true + sum := hex.EncodeToString(hasher.Sum(nil)) + if sum != d.sha256Sum { + d.handleError(ErrSHA256Mismatch) + } else { + d.statsCh <- stats + } + return + } + d.handleError(fmt.Errorf("error reading response body: %v", err)) + return + } + } + } +} + +func (d *Downloader) handleError(err error) { + select { + case d.errCh <- err: + default: + } + d.Stop() +} diff --git a/util/downloader/downloader_test.go b/util/downloader/downloader_test.go new file mode 100644 index 000000000..f7fc3b231 --- /dev/null +++ b/util/downloader/downloader_test.go @@ -0,0 +1,126 @@ +package downloader + +import ( + "context" + "github.com/stretchr/testify/assert" + "log" + "os" + "path/filepath" + "sync" + "testing" + "time" +) + +func setup() *Downloader { + fileURL := "https://github.com/pactus-project/Whitepaper/releases/latest/download/pactus_whitepaper.pdf" + filePath := "./testdata/example.pdf" + expectedSHA256 := "ea956128717b49669f29eeed116bc11b9bbdcd50f1df130e124ffd36afe71652" + + dl := NewDownloader(fileURL, filePath, expectedSHA256) + return dl +} + +func cleanup(path string) error { + err := os.Remove(path) + if err != nil { + return err + } + dir := filepath.Dir(path) + err = os.RemoveAll(dir) + if err != nil { + return err + } + return nil +} + +func TestDownloader(t *testing.T) { + dl := setup() + + assrt := assert.New(t) + var wg sync.WaitGroup + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + wg.Add(1) + go func() { + defer wg.Done() + dl.Start() + }() + + go func() { + select { + case <-ctx.Done(): + dl.Stop() + assrt.Fail("Download test timed out") + } + }() + + done := make(chan bool) + + go func() { + for stat := range dl.Stats() { + log.Printf("Downloaded: %d / %d (%.2f%%)\n", stat.Downloaded, stat.TotalSize, stat.Percent) + assrt.True(stat.Downloaded <= stat.TotalSize, "Downloaded size should not exceed total size") + assrt.True(stat.Percent <= 100, "Download percentage should not exceed 100") + + if stat.Completed { + log.Println("Download completed successfully") + assrt.Equal(float64(100), stat.Percent, "Download should be 100% complete") + done <- true + return + } + } + }() + + go func() { + for err := range dl.Errors() { + assrt.Fail("Download encountered an error", err) + done <- true + return + } + }() + + select { + case <-done: + case <-time.After(2 * time.Minute): + dl.Stop() + assrt.Fail("Download test timed out") + } + + assert.NoError(t, cleanup(dl.filePath)) + + wg.Wait() +} + +func TestDownloaderOperations(t *testing.T) { + dl := setup() + dl.Start() + + assrt := assert.New(t) + + go func() { + for stat := range dl.Stats() { + log.Printf("Downloaded: %d / %d (%.2f%%)\n", stat.Downloaded, stat.TotalSize, stat.Percent) + assrt.True(stat.Downloaded <= stat.TotalSize, "Downloaded size should not exceed total size") + assrt.True(stat.Percent <= 100, "Download percentage should not exceed 100") + + if stat.Completed { + log.Println("Download completed successfully") + assrt.Equal(float64(100), stat.Percent, "Download should be 100% complete") + return + } + } + }() + + time.Sleep(1 * time.Second) + dl.Pause() + t.Log("Paused") + time.Sleep(3 * time.Second) + dl.Resume() + t.Log("Resumed") + + time.Sleep(1 * time.Second) + dl.Stop() + + assrt.NoError(cleanup(dl.filePath)) +} From 791c9f899cc840f9ad6c37439875c96e5dac8312 Mon Sep 17 00:00:00 2001 From: Javad Date: Tue, 9 Jul 2024 13:36:56 +0330 Subject: [PATCH 2/9] fix: lint errors --- util/downloader/downloader.go | 213 ++++++++++++++++++++--------- util/downloader/downloader_test.go | 21 ++- 2 files changed, 160 insertions(+), 74 deletions(-) diff --git a/util/downloader/downloader.go b/util/downloader/downloader.go index 911e22f5a..2d9513095 100644 --- a/util/downloader/downloader.go +++ b/util/downloader/downloader.go @@ -1,10 +1,12 @@ package downloader import ( + "context" "crypto/sha256" "encoding/hex" "errors" "fmt" + "hash" "io" "net/http" "os" @@ -16,14 +18,13 @@ var ( ErrHeaderRequest = errors.New("request header error") ErrSHA256Mismatch = errors.New("sha256 mismatch") ErrCreateDir = errors.New("create dir error") - ErrOpenFile = errors.New("open file error") ErrInvalidFilePath = errors.New("file path is a directory, not a file") ErrGetFileInfo = errors.New("get file info error") ErrCopyExistsFileData = errors.New("error copying existing file data") ErrDoRequest = errors.New("error doing request") ErrFileWriting = errors.New("error writing file") ErrNewRequest = errors.New("error creating request") - ErrOpenFileExists = errors.New("error opening file exists") + ErrOpenFileExists = errors.New("error opening existing file") ) type Downloader struct { @@ -32,9 +33,9 @@ type Downloader struct { sha256Sum string fileType string fileName string - pause chan bool - resume chan bool - stop chan bool + pause chan struct{} + resume chan struct{} + stop chan struct{} statsCh chan Stats errCh chan error wg sync.WaitGroup @@ -47,14 +48,14 @@ type Stats struct { Completed bool } -func NewDownloader(url, filePath, sha256Sum string) *Downloader { +func New(url, filePath, sha256Sum string) *Downloader { return &Downloader{ url: url, filePath: filePath, sha256Sum: sha256Sum, - pause: make(chan bool), - resume: make(chan bool), - stop: make(chan bool), + pause: make(chan struct{}), + resume: make(chan struct{}), + stop: make(chan struct{}), statsCh: make(chan Stats), errCh: make(chan error, 1), } @@ -62,19 +63,19 @@ func NewDownloader(url, filePath, sha256Sum string) *Downloader { func (d *Downloader) Start() { d.wg.Add(1) - go d.download() + go d.download(context.Background()) } func (d *Downloader) Pause() { - d.pause <- true + d.pause <- struct{}{} } func (d *Downloader) Resume() { - d.resume <- true + d.resume <- struct{}{} } func (d *Downloader) Stop() { - d.stop <- true + d.stop <- struct{}{} d.wg.Wait() close(d.statsCh) close(d.errCh) @@ -96,118 +97,196 @@ func (d *Downloader) Errors() <-chan error { return d.errCh } -func (d *Downloader) download() { +func (d *Downloader) download(ctx context.Context) { defer d.wg.Done() - resp, err := http.Head(d.url) + stats, err := d.getHeader(ctx) if err != nil { - d.handleError(ErrHeaderRequest) - return - } + d.handleError(err) - stats := Stats{ - TotalSize: resp.ContentLength, + return } - d.fileType = resp.Header.Get("Content-Type") d.fileName = filepath.Base(d.filePath) + if err := d.createDir(); err != nil { + d.handleError(err) - dir := filepath.Dir(d.filePath) - if err := os.MkdirAll(dir, 0755); err != nil { - d.handleError(ErrCreateDir) return } - fileInfo, err := os.Stat(d.filePath) - if err == nil && fileInfo.IsDir() { - d.handleError(ErrInvalidFilePath) + out, err := d.openFile() + if err != nil { + d.handleError(err) + return } + defer func() { + _ = out.Close() + }() + + if err := d.validateExistingFile(out, &stats); err != nil { + d.handleError(err) - out, err := os.OpenFile(d.filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { - d.handleError(ErrOpenFile) return } - defer out.Close() - fileInfo, err = out.Stat() + if err := d.downloadFile(ctx, out, &stats); err != nil { + d.handleError(err) + } +} + +func (d *Downloader) getHeader(ctx context.Context) (Stats, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, d.url, http.NoBody) if err != nil { - d.handleError(ErrGetFileInfo) - return + return Stats{}, ErrHeaderRequest } - stats.Downloaded = fileInfo.Size() - req, err := http.NewRequest("GET", d.url, nil) + resp, err := http.DefaultClient.Do(req) if err != nil { - d.handleError(ErrNewRequest) - return + return Stats{}, ErrHeaderRequest + } + + defer func() { + _ = resp.Body.Close() + }() + + d.fileType = resp.Header.Get("Content-Type") + + return Stats{ + TotalSize: resp.ContentLength, + }, nil +} + +func (d *Downloader) createDir() error { + dir := filepath.Dir(d.filePath) + if err := os.MkdirAll(dir, 0o750); err != nil { + return ErrCreateDir + } + + return nil +} + +func (d *Downloader) openFile() (*os.File, error) { + fileInfo, err := os.Stat(d.filePath) + if err == nil && fileInfo.IsDir() { + return nil, ErrInvalidFilePath } - if stats.Downloaded > 0 { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", stats.Downloaded)) + + return os.OpenFile(d.filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) +} + +func (*Downloader) validateExistingFile(out *os.File, stats *Stats) error { + fileInfo, err := out.Stat() + if err != nil { + return ErrGetFileInfo } + stats.Downloaded = fileInfo.Size() + return nil +} + +func (d *Downloader) downloadFile(ctx context.Context, out *os.File, stats *Stats) error { client := &http.Client{} - resp, err = client.Do(req) + req, err := d.createRequest(ctx, stats.Downloaded) if err != nil { - d.handleError(ErrDoRequest) - return + return err } - defer resp.Body.Close() + + resp, err := client.Do(req) + if err != nil { + return ErrDoRequest + } + + defer func() { + _ = resp.Body.Close() + }() buffer := make([]byte, 32*1024) hasher := sha256.New() - // Update the hasher with the already downloaded part - if stats.Downloaded > 0 { + if err := d.updateHasherWithExistingData(stats.Downloaded, hasher); err != nil { + return err + } + + return d.writeToFile(resp, out, buffer, hasher, stats) +} + +func (d *Downloader) createRequest(ctx context.Context, downloaded int64) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, d.url, http.NoBody) + if err != nil { + return nil, ErrNewRequest + } + if downloaded > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", downloaded)) + } + + return req, nil +} + +func (d *Downloader) updateHasherWithExistingData(downloaded int64, hasher io.Writer) error { + if downloaded > 0 { existingFile, err := os.Open(d.filePath) if err != nil { - d.handleError(ErrOpenFileExists) - return + return ErrOpenFileExists } - defer existingFile.Close() - if _, err := io.CopyN(hasher, existingFile, stats.Downloaded); err != nil { - d.handleError(ErrCopyExistsFileData) - return + defer func() { + _ = existingFile.Close() + }() + + if _, err := io.CopyN(hasher, existingFile, downloaded); err != nil { + return ErrCopyExistsFileData } } + return nil +} + +func (d *Downloader) writeToFile(resp *http.Response, out *os.File, buffer []byte, + hasher hash.Hash, stats *Stats, +) error { for { select { case <-d.pause: <-d.resume case <-d.stop: - return + return nil default: n, err := resp.Body.Read(buffer) if n > 0 { if _, err := out.Write(buffer[:n]); err != nil { - d.handleError(ErrFileWriting) - return + return ErrFileWriting } - hasher.Write(buffer[:n]) + + if _, err := hasher.Write(buffer[:n]); err != nil { + return ErrFileWriting + } + stats.Downloaded += int64(n) stats.Percent = float64(stats.Downloaded) / float64(stats.TotalSize) * 100 - d.statsCh <- stats + d.statsCh <- *stats } if err != nil { if err == io.EOF { - stats.Completed = true - sum := hex.EncodeToString(hasher.Sum(nil)) - if sum != d.sha256Sum { - d.handleError(ErrSHA256Mismatch) - } else { - d.statsCh <- stats - } - return + return d.finalizeDownload(hasher, stats) } - d.handleError(fmt.Errorf("error reading response body: %v", err)) - return + + return fmt.Errorf("error reading response body: %w", err) } } } } +func (d *Downloader) finalizeDownload(hasher hash.Hash, stats *Stats) error { + stats.Completed = true + sum := hex.EncodeToString(hasher.Sum(nil)) + if sum != d.sha256Sum { + return ErrSHA256Mismatch + } + d.statsCh <- *stats + + return nil +} + func (d *Downloader) handleError(err error) { select { case d.errCh <- err: diff --git a/util/downloader/downloader_test.go b/util/downloader/downloader_test.go index f7fc3b231..84adc0dbd 100644 --- a/util/downloader/downloader_test.go +++ b/util/downloader/downloader_test.go @@ -2,13 +2,14 @@ package downloader import ( "context" - "github.com/stretchr/testify/assert" "log" "os" "path/filepath" "sync" "testing" "time" + + "github.com/stretchr/testify/assert" ) func setup() *Downloader { @@ -16,7 +17,8 @@ func setup() *Downloader { filePath := "./testdata/example.pdf" expectedSHA256 := "ea956128717b49669f29eeed116bc11b9bbdcd50f1df130e124ffd36afe71652" - dl := NewDownloader(fileURL, filePath, expectedSHA256) + dl := New(fileURL, filePath, expectedSHA256) + return dl } @@ -30,6 +32,7 @@ func cleanup(path string) error { if err != nil { return err } + return nil } @@ -48,11 +51,9 @@ func TestDownloader(t *testing.T) { }() go func() { - select { - case <-ctx.Done(): - dl.Stop() - assrt.Fail("Download test timed out") - } + <-ctx.Done() + dl.Stop() + assrt.Fail("Download test timed out") }() done := make(chan bool) @@ -67,6 +68,7 @@ func TestDownloader(t *testing.T) { log.Println("Download completed successfully") assrt.Equal(float64(100), stat.Percent, "Download should be 100% complete") done <- true + return } } @@ -76,6 +78,7 @@ func TestDownloader(t *testing.T) { for err := range dl.Errors() { assrt.Fail("Download encountered an error", err) done <- true + return } }() @@ -87,6 +90,9 @@ func TestDownloader(t *testing.T) { assrt.Fail("Download test timed out") } + t.Log(dl.FileName()) + t.Log(dl.FileType()) + assert.NoError(t, cleanup(dl.filePath)) wg.Wait() @@ -107,6 +113,7 @@ func TestDownloaderOperations(t *testing.T) { if stat.Completed { log.Println("Download completed successfully") assrt.Equal(float64(100), stat.Percent, "Download should be 100% complete") + return } } From 8649c7786b61c325cb3c65c6cd32b1a2b5bbfdd5 Mon Sep 17 00:00:00 2001 From: Javad Date: Wed, 17 Jul 2024 11:03:11 +0330 Subject: [PATCH 3/9] fix: remove stop, resume and pause operation --- util/downloader/downloader.go | 95 ++++++++++++++--------------------- 1 file changed, 38 insertions(+), 57 deletions(-) diff --git a/util/downloader/downloader.go b/util/downloader/downloader.go index 2d9513095..89f07e948 100644 --- a/util/downloader/downloader.go +++ b/util/downloader/downloader.go @@ -11,7 +11,6 @@ import ( "net/http" "os" "path/filepath" - "sync" ) var ( @@ -28,17 +27,14 @@ var ( ) type Downloader struct { + client *http.Client url string filePath string sha256Sum string fileType string fileName string - pause chan struct{} - resume chan struct{} - stop chan struct{} statsCh chan Stats errCh chan error - wg sync.WaitGroup } type Stats struct { @@ -48,37 +44,25 @@ type Stats struct { Completed bool } -func New(url, filePath, sha256Sum string) *Downloader { +func New(url, filePath, sha256Sum string, opts ...Option) *Downloader { + opt := defaultOptions() + + for _, o := range opts { + o(opt) + } + return &Downloader{ + client: opt.client, url: url, filePath: filePath, sha256Sum: sha256Sum, - pause: make(chan struct{}), - resume: make(chan struct{}), - stop: make(chan struct{}), statsCh: make(chan Stats), errCh: make(chan error, 1), } } -func (d *Downloader) Start() { - d.wg.Add(1) - go d.download(context.Background()) -} - -func (d *Downloader) Pause() { - d.pause <- struct{}{} -} - -func (d *Downloader) Resume() { - d.resume <- struct{}{} -} - -func (d *Downloader) Stop() { - d.stop <- struct{}{} - d.wg.Wait() - close(d.statsCh) - close(d.errCh) +func (d *Downloader) Start(ctx context.Context) { + go d.download(ctx) } func (d *Downloader) Stats() <-chan Stats { @@ -98,8 +82,6 @@ func (d *Downloader) Errors() <-chan error { } func (d *Downloader) download(ctx context.Context) { - defer d.wg.Done() - stats, err := d.getHeader(ctx) if err != nil { d.handleError(err) @@ -141,7 +123,7 @@ func (d *Downloader) getHeader(ctx context.Context) (Stats, error) { return Stats{}, ErrHeaderRequest } - resp, err := http.DefaultClient.Do(req) + resp, err := d.client.Do(req) if err != nil { return Stats{}, ErrHeaderRequest } @@ -186,13 +168,12 @@ func (*Downloader) validateExistingFile(out *os.File, stats *Stats) error { } func (d *Downloader) downloadFile(ctx context.Context, out *os.File, stats *Stats) error { - client := &http.Client{} req, err := d.createRequest(ctx, stats.Downloaded) if err != nil { return err } - resp, err := client.Do(req) + resp, err := d.client.Do(req) if err != nil { return ErrDoRequest } @@ -245,33 +226,26 @@ func (d *Downloader) writeToFile(resp *http.Response, out *os.File, buffer []byt hasher hash.Hash, stats *Stats, ) error { for { - select { - case <-d.pause: - <-d.resume - case <-d.stop: - return nil - default: - n, err := resp.Body.Read(buffer) - if n > 0 { - if _, err := out.Write(buffer[:n]); err != nil { - return ErrFileWriting - } - - if _, err := hasher.Write(buffer[:n]); err != nil { - return ErrFileWriting - } - - stats.Downloaded += int64(n) - stats.Percent = float64(stats.Downloaded) / float64(stats.TotalSize) * 100 - d.statsCh <- *stats + n, err := resp.Body.Read(buffer) + if n > 0 { + if _, err := out.Write(buffer[:n]); err != nil { + return ErrFileWriting + } + + if _, err := hasher.Write(buffer[:n]); err != nil { + return ErrFileWriting } - if err != nil { - if err == io.EOF { - return d.finalizeDownload(hasher, stats) - } - return fmt.Errorf("error reading response body: %w", err) + stats.Downloaded += int64(n) + stats.Percent = float64(stats.Downloaded) / float64(stats.TotalSize) * 100 + d.statsCh <- *stats + } + if err != nil { + if err == io.EOF { + return d.finalizeDownload(hasher, stats) } + + return fmt.Errorf("error reading response body: %w", err) } } } @@ -284,13 +258,20 @@ func (d *Downloader) finalizeDownload(hasher hash.Hash, stats *Stats) error { } d.statsCh <- *stats + d.stop() + return nil } +func (d *Downloader) stop() { + close(d.statsCh) + close(d.errCh) +} + func (d *Downloader) handleError(err error) { select { case d.errCh <- err: default: + d.stop() } - d.Stop() } From d4eac28012fbef3cde07b0fcff84d5bdc56ecccc Mon Sep 17 00:00:00 2001 From: Javad Date: Wed, 17 Jul 2024 11:03:30 +0330 Subject: [PATCH 4/9] fix: used http test server for download test --- util/downloader/downloader_test.go | 91 +++++++++++------------------- 1 file changed, 32 insertions(+), 59 deletions(-) diff --git a/util/downloader/downloader_test.go b/util/downloader/downloader_test.go index 84adc0dbd..d9cb352d7 100644 --- a/util/downloader/downloader_test.go +++ b/util/downloader/downloader_test.go @@ -2,26 +2,19 @@ package downloader import ( "context" + "crypto/sha256" + "encoding/hex" "log" + "net/http" + "net/http/httptest" "os" "path/filepath" - "sync" "testing" "time" "github.com/stretchr/testify/assert" ) -func setup() *Downloader { - fileURL := "https://github.com/pactus-project/Whitepaper/releases/latest/download/pactus_whitepaper.pdf" - filePath := "./testdata/example.pdf" - expectedSHA256 := "ea956128717b49669f29eeed116bc11b9bbdcd50f1df130e124ffd36afe71652" - - dl := New(fileURL, filePath, expectedSHA256) - - return dl -} - func cleanup(path string) error { err := os.Remove(path) if err != nil { @@ -37,23 +30,35 @@ func cleanup(path string) error { } func TestDownloader(t *testing.T) { - dl := setup() + fileContent := []byte("This is a test file content") + fileURL := "/testfile" + expectedSHA256 := sha256.Sum256(fileContent) + expectedSHA256Hex := hex.EncodeToString(expectedSHA256[:]) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == fileURL { + _, err := w.Write(fileContent) + assert.NoError(t, err) + } else { + http.NotFound(w, r) + } + })) + defer server.Close() + + filePath := "./testdata/example_testfile.txt" + + defer func() { + assert.NoError(t, os.RemoveAll("./testdata")) + }() + + dl := New(server.URL+fileURL, filePath, expectedSHA256Hex, WithCustomClient(server.Client())) assrt := assert.New(t) - var wg sync.WaitGroup ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - wg.Add(1) go func() { - defer wg.Done() - dl.Start() - }() - - go func() { - <-ctx.Done() - dl.Stop() - assrt.Fail("Download test timed out") + dl.Start(ctx) }() done := make(chan bool) @@ -86,48 +91,16 @@ func TestDownloader(t *testing.T) { select { case <-done: case <-time.After(2 * time.Minute): - dl.Stop() + cancel() assrt.Fail("Download test timed out") } t.Log(dl.FileName()) t.Log(dl.FileType()) - assert.NoError(t, cleanup(dl.filePath)) - - wg.Wait() -} - -func TestDownloaderOperations(t *testing.T) { - dl := setup() - dl.Start() - - assrt := assert.New(t) + downloadedContent, err := os.ReadFile(filePath) + assrt.NoError(err, "Failed to read the downloaded file") + assrt.Equal(fileContent, downloadedContent, "Downloaded file content does not match expected content") - go func() { - for stat := range dl.Stats() { - log.Printf("Downloaded: %d / %d (%.2f%%)\n", stat.Downloaded, stat.TotalSize, stat.Percent) - assrt.True(stat.Downloaded <= stat.TotalSize, "Downloaded size should not exceed total size") - assrt.True(stat.Percent <= 100, "Download percentage should not exceed 100") - - if stat.Completed { - log.Println("Download completed successfully") - assrt.Equal(float64(100), stat.Percent, "Download should be 100% complete") - - return - } - } - }() - - time.Sleep(1 * time.Second) - dl.Pause() - t.Log("Paused") - time.Sleep(3 * time.Second) - dl.Resume() - t.Log("Resumed") - - time.Sleep(1 * time.Second) - dl.Stop() - - assrt.NoError(cleanup(dl.filePath)) + assert.NoError(t, cleanup(dl.filePath)) } From cec833c2633e500cd965c42e3051afdf54f0707e Mon Sep 17 00:00:00 2001 From: Javad Date: Wed, 17 Jul 2024 11:09:12 +0330 Subject: [PATCH 5/9] fix: add custom options for downloader --- util/downloader/options.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 util/downloader/options.go diff --git a/util/downloader/options.go b/util/downloader/options.go new file mode 100644 index 000000000..27c0042f4 --- /dev/null +++ b/util/downloader/options.go @@ -0,0 +1,21 @@ +package downloader + +import "net/http" + +type options struct { + client *http.Client +} + +type Option func(*options) + +func defaultOptions() *options { + return &options{ + client: http.DefaultClient, + } +} + +func WithCustomClient(client *http.Client) Option { + return func(o *options) { + o.client = client + } +} From b055cdcd3fb261c665dd17b1ffa6fe97fdd6e083 Mon Sep 17 00:00:00 2001 From: Javad Date: Wed, 17 Jul 2024 11:19:24 +0330 Subject: [PATCH 6/9] fix: get context cancel signal in write file loop for canceling --- util/downloader/downloader.go | 45 ++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/util/downloader/downloader.go b/util/downloader/downloader.go index 89f07e948..bae58fe09 100644 --- a/util/downloader/downloader.go +++ b/util/downloader/downloader.go @@ -189,7 +189,7 @@ func (d *Downloader) downloadFile(ctx context.Context, out *os.File, stats *Stat return err } - return d.writeToFile(resp, out, buffer, hasher, stats) + return d.writeToFile(ctx, resp, out, buffer, hasher, stats) } func (d *Downloader) createRequest(ctx context.Context, downloaded int64) (*http.Request, error) { @@ -222,30 +222,37 @@ func (d *Downloader) updateHasherWithExistingData(downloaded int64, hasher io.Wr return nil } -func (d *Downloader) writeToFile(resp *http.Response, out *os.File, buffer []byte, +func (d *Downloader) writeToFile(ctx context.Context, resp *http.Response, out *os.File, buffer []byte, hasher hash.Hash, stats *Stats, ) error { for { - n, err := resp.Body.Read(buffer) - if n > 0 { - if _, err := out.Write(buffer[:n]); err != nil { - return ErrFileWriting + select { + case <-ctx.Done(): + d.stop() + + return ctx.Err() + default: + n, err := resp.Body.Read(buffer) + if n > 0 { + if _, err := out.Write(buffer[:n]); err != nil { + return ErrFileWriting + } + + if _, err := hasher.Write(buffer[:n]); err != nil { + return ErrFileWriting + } + + stats.Downloaded += int64(n) + stats.Percent = float64(stats.Downloaded) / float64(stats.TotalSize) * 100 + d.statsCh <- *stats } + if err != nil { + if err == io.EOF { + return d.finalizeDownload(hasher, stats) + } - if _, err := hasher.Write(buffer[:n]); err != nil { - return ErrFileWriting + return fmt.Errorf("error reading response body: %w", err) } - - stats.Downloaded += int64(n) - stats.Percent = float64(stats.Downloaded) / float64(stats.TotalSize) * 100 - d.statsCh <- *stats - } - if err != nil { - if err == io.EOF { - return d.finalizeDownload(hasher, stats) - } - - return fmt.Errorf("error reading response body: %w", err) } } } From 32b95c49380f4bd961be76162a7407ce8d4c6706 Mon Sep 17 00:00:00 2001 From: Javad Date: Wed, 17 Jul 2024 11:21:06 +0330 Subject: [PATCH 7/9] fix: remove cleanup function in test --- util/downloader/downloader_test.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/util/downloader/downloader_test.go b/util/downloader/downloader_test.go index d9cb352d7..8b4e04433 100644 --- a/util/downloader/downloader_test.go +++ b/util/downloader/downloader_test.go @@ -8,27 +8,12 @@ import ( "net/http" "net/http/httptest" "os" - "path/filepath" "testing" "time" "github.com/stretchr/testify/assert" ) -func cleanup(path string) error { - err := os.Remove(path) - if err != nil { - return err - } - dir := filepath.Dir(path) - err = os.RemoveAll(dir) - if err != nil { - return err - } - - return nil -} - func TestDownloader(t *testing.T) { fileContent := []byte("This is a test file content") fileURL := "/testfile" @@ -101,6 +86,4 @@ func TestDownloader(t *testing.T) { downloadedContent, err := os.ReadFile(filePath) assrt.NoError(err, "Failed to read the downloaded file") assrt.Equal(fileContent, downloadedContent, "Downloaded file content does not match expected content") - - assert.NoError(t, cleanup(dl.filePath)) } From 842c3675417710a5cd0643939ed43a48b597856d Mon Sep 17 00:00:00 2001 From: Javad Date: Wed, 17 Jul 2024 12:41:34 +0330 Subject: [PATCH 8/9] fix: used temp file util for filePath in test --- util/downloader/downloader_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/downloader/downloader_test.go b/util/downloader/downloader_test.go index 8b4e04433..881584885 100644 --- a/util/downloader/downloader_test.go +++ b/util/downloader/downloader_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/pactus-project/pactus/util" "github.com/stretchr/testify/assert" ) @@ -30,7 +31,7 @@ func TestDownloader(t *testing.T) { })) defer server.Close() - filePath := "./testdata/example_testfile.txt" + filePath := util.TempFilePath() defer func() { assert.NoError(t, os.RemoveAll("./testdata")) From b18b633368b33ef50ae5f981cbe535a3b52e7183 Mon Sep 17 00:00:00 2001 From: Javad Date: Wed, 17 Jul 2024 13:50:26 +0330 Subject: [PATCH 9/9] fix: remove ignored testdata path downloader --- .gitignore | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitignore b/.gitignore index a2133b490..2a955a371 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,3 @@ todo # VIM artifacts .swp .*.sw* - -util/downloader/testdata/