From 9853a2f5557c2e5b6f065734ce4f001d7af80b60 Mon Sep 17 00:00:00 2001 From: Bob Gordon Date: Tue, 20 Mar 2018 08:06:05 +0000 Subject: [PATCH] add minio support via http --- README.md | 5 ++ cmd/imageproxy/main.go | 3 + internal/miniocache/miniocache.go | 120 ++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 internal/miniocache/miniocache.go diff --git a/README.md b/README.md index 606f62e67..53d5b8cbd 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,11 @@ enabled using the `-cache` flag. It supports the following values: environmental variables be set. (Additional methods of loading credentials are documented in the [aws-sdk-go session package](https://docs.aws.amazon.com/sdk-for-go/api/aws/session/)). + - http (Minio) URL (e.g. `http://endpoint/region/bucket/optional-path-prefix`) - will cache + images on Minio S3. This requires `AWS_ACCESS_KEY_ID` and `AWS_SECRET_KEY` + environmental variables be set. (Additional methods of loading credentials + are documented in the [aws-sdk-go session + package](https://docs.aws.amazon.com/sdk-for-go/api/aws/session/)). - gcs URL (e.g. `gcs://bucket-name/optional-path-prefix`) - will cache images on Google Cloud Storage. Authentication is documented in Google's [Application Default Credentials diff --git a/cmd/imageproxy/main.go b/cmd/imageproxy/main.go index 966d6b652..5158be9fd 100644 --- a/cmd/imageproxy/main.go +++ b/cmd/imageproxy/main.go @@ -36,6 +36,7 @@ import ( "github.com/peterbourgon/diskv" "willnorris.com/go/imageproxy" "willnorris.com/go/imageproxy/internal/gcscache" + "willnorris.com/go/imageproxy/internal/miniocache" "willnorris.com/go/imageproxy/internal/s3cache" ) @@ -153,6 +154,8 @@ func parseCache(c string) (imageproxy.Cache, error) { return rediscache.NewWithClient(conn), nil case "s3": return s3cache.New(u.String()) + case "http": + return miniocache.New(u.String()) case "file": fallthrough default: diff --git a/internal/miniocache/miniocache.go b/internal/miniocache/miniocache.go new file mode 100644 index 000000000..fa660ee33 --- /dev/null +++ b/internal/miniocache/miniocache.go @@ -0,0 +1,120 @@ +// Package miniocache provides an httpcache.Cache implementation that stores +// cached values on Minio S3. +package miniocache + +import ( + "bytes" + "crypto/md5" + "encoding/hex" + "io" + "io/ioutil" + "log" + "net/url" + "path" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +type cache struct { + *s3.S3 + bucket, prefix string +} + +func (c *cache) Get(key string) ([]byte, bool) { + key = path.Join(c.prefix, keyToFilename(key)) + input := &s3.GetObjectInput{ + Bucket: &c.bucket, + Key: &key, + } + + resp, err := c.GetObject(input) + if err != nil { + if aerr, ok := err.(awserr.Error); ok && aerr.Code() != "NoSuchKey" { + log.Printf("error fetching from minio: %v", aerr) + } + return nil, false + } + + value, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Printf("error reading minio response body: %v", err) + return nil, false + } + + return value, true +} +func (c *cache) Set(key string, value []byte) { + key = path.Join(c.prefix, keyToFilename(key)) + input := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(bytes.NewReader(value)), + Bucket: &c.bucket, + Key: &key, + } + + _, err := c.PutObject(input) + if err != nil { + log.Printf("error writing to minio: %v", err) + } +} +func (c *cache) Delete(key string) { + key = path.Join(c.prefix, keyToFilename(key)) + input := &s3.DeleteObjectInput{ + Bucket: &c.bucket, + Key: &key, + } + + _, err := c.DeleteObject(input) + if err != nil { + log.Printf("error deleting from minio: %v", err) + } +} + +func keyToFilename(key string) string { + h := md5.New() + io.WriteString(h, key) + return hex.EncodeToString(h.Sum(nil)) +} + +// New constructs a cache configured using the provided URL string. URL should +// be of the form: "http://endpoint/region/bucket/optional-path-prefix". Credentials +// should be specified using one of the mechanisms supported by aws-sdk-go (see +// https://docs.aws.amazon.com/sdk-for-go/api/aws/session/). +func New(s string) (*cache, error) { + u, err := url.Parse(s) + if err != nil { + return nil, err + } + + endpoint := u.Host + path := strings.SplitN(strings.TrimPrefix(u.Path, "/"), "/", 2) + region := path[0] + bucket := path[1] + + var prefix string + if len(path) > 1 { + prefix = path[2] + } + + // Configure to use Minio Server + s3Config := &aws.Config{ + Endpoint: aws.String(endpoint), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + + sess, err := session.NewSession(s3Config) + if err != nil { + return nil, err + } + + return &cache{ + S3: s3.New(sess), + bucket: bucket, + prefix: prefix, + }, nil +}