Skip to content

Commit

Permalink
example/service/s3/putObjectWithProgress: fix example for file upload…
Browse files Browse the repository at this point in the history
… with progress (#3377)

Fixes #2468 by ignoring the first read of the progress reader wrapper. Since the first read is used for signing the request, not upload progress.
mayongze authored Aug 10, 2020

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 827e841 commit eb7121d
Showing 3 changed files with 36 additions and 35 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG_PENDING.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
### SDK Features

### SDK Enhancements
* `example/service/s3/putObjectWithProgress`: Fix example for file upload with progress ([#3377](https://github.com/aws/aws-sdk-go/pull/3377))
* Fixes [#2468](https://github.com/aws/aws-sdk-go/issues/2468) by ignoring the first read of the progress reader wrapper. Since the first read is used for signing the request, not upload progress.
* Updated the example to write progress inline instead of newlines.
* `service/dynamodb/dynamodbattribute`: Fix typo in package docs ([#3446](https://github.com/aws/aws-sdk-go/pull/3446))
* Fixes typo in dynamodbattribute package docs.

2 changes: 1 addition & 1 deletion example/service/s3/putObjectWithProcess/README.md
Original file line number Diff line number Diff line change
@@ -10,5 +10,5 @@ The example uses the bucket name provided, one key for object, and output the pr
The Object size should larger than 5M or your will not see the progress

```sh
AWS_REGION=<region> go run putObjWithProcess.go <credential> <bucket> <key for object> <local file name>
AWS_REGION=<region> go run -tags example putObjWithProcess.go <bucket> <key for object> <local file name>
```
66 changes: 32 additions & 34 deletions example/service/s3/putObjectWithProcess/putObjWithProcess.go
Original file line number Diff line number Diff line change
@@ -3,20 +3,22 @@
package main

import (
"fmt"
"log"
"os"
"sync/atomic"
"sync"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

type CustomReader struct {
fp *os.File
size int64
read int64
fp *os.File
size int64
read int64
signMap map[int64]struct{}
mux sync.Mutex
}

func (r *CustomReader) Read(p []byte) (int, error) {
@@ -29,14 +31,16 @@ func (r *CustomReader) ReadAt(p []byte, off int64) (int, error) {
return n, err
}

// Got the length have read( or means has uploaded), and you can construct your message
atomic.AddInt64(&r.read, int64(n))

// I have no idea why the read length need to be div 2,
// maybe the request read once when Sign and actually send call ReadAt again
// It works for me
log.Printf("total read:%d progress:%d%%\n", r.read/2, int(float32(r.read*100/2)/float32(r.size)))

r.mux.Lock()
// Ignore the first signature call
if _, ok := r.signMap[off]; ok {
// Got the length have read( or means has uploaded), and you can construct your message
r.read += int64(n)
fmt.Printf("\rtotal read:%d progress:%d%%", r.read, int(float32(r.read*100)/float32(r.size)))
} else {
r.signMap[off] = struct{}{}
}
r.mux.Unlock()
return n, err
}

@@ -46,40 +50,34 @@ func (r *CustomReader) Seek(offset int64, whence int) (int64, error) {

func main() {
if len(os.Args) < 4 {
log.Println("USAGE ERROR: AWS_REGION=us-east-1 go run putObjWithProcess.go <credential> <bucket> <key for object> <local file name>")
log.Println("USAGE ERROR: AWS_REGION=us-west-2 go run -tags example putObjWithProcess.go <bucket> <key for object> <local file name>")
return
}

credential := os.Args[1]
bucket := os.Args[2]
key := os.Args[3]
fileName := os.Args[4]
bucket := os.Args[1]
key := os.Args[2]
filename := os.Args[3]

creds := credentials.NewSharedCredentials(credential, "default")
if _, err := creds.Get(); err != nil {
log.Println("ERROR:", err)
return
sess, err := session.NewSession()
if err != nil {
log.Fatalf("failed to load session, %v", err)
}

sess := session.New(&aws.Config{
Credentials: creds,
})

file, err := os.Open(fileName)
file, err := os.Open(filename)
if err != nil {
log.Println("ERROR:", err)
return
log.Fatalf("failed to open file %v, %v", filename, err)
}

fileInfo, err := file.Stat()
if err != nil {
log.Println("ERROR:", err)
log.Fatalf("failed to stat file %v, %v", filename, err)
return
}

reader := &CustomReader{
fp: file,
size: fileInfo.Size(),
fp: file,
size: fileInfo.Size(),
signMap: map[int64]struct{}{},
}

uploader := s3manager.NewUploader(sess, func(u *s3manager.Uploader) {
@@ -92,11 +90,11 @@ func main() {
Key: aws.String(key),
Body: reader,
})

if err != nil {
log.Println("ERROR:", err)
log.Fatalf("failed to put file %v, %v", filename, err)
return
}

fmt.Println()
log.Println(output.Location)
}

0 comments on commit eb7121d

Please sign in to comment.