Add proxied file system blob implementation with its route handler

.github/workflows/build.yml

@@ -37,4 +37,6 @@ jobs:
         S3_BASE_URL: ${{ secrets.S3_BASE_URL }}
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
+
+    - name: Test pfsblob
+      run: go run github.com/Shivam010/upload/pfsblob/tests

README.md

@@ -1,15 +1,44 @@
-# upload 
+# upload
+
 [![Build](https://github.com/Shivam010/upload/actions/workflows/build.yml/badge.svg)](https://github.com/Shivam010/upload/actions/workflows/build.yml)
 [![Go Report Card](https://goreportcard.com/badge/github.com/Shivam010/upload?dropcache)](https://goreportcard.com/report/github.com/Shivam010/upload)
 [![Go Reference](https://pkg.go.dev/badge/github.com/Shivam010/upload)](https://pkg.go.dev/github.com/Shivam010/upload)
 [![License](https://img.shields.io/badge/license-MIT-mildgreen.svg)](https://github.com/Shivam010/upload/blob/master/LICENSE)
 [![GitHub release](https://img.shields.io/github/release/Shivam010/upload.svg)](https://github.com/Shivam010/upload/releases)
 
-Package upload provides an easy and portable way to interact with CDN, buckets or blobs within any cloud/local storage location. And provides methods to read or write or upload or delete files to blob storage on GCP, AWS, Azure, in-memory, local and more.
+Package upload provides an easy and portable way to interact with CDN, buckets or blobs within any cloud/local storage
+location. And provides methods to read or write or upload or delete files to blob storage on GCP, AWS, Azure, in-memory,
+local and more.
 
 It wraps `gocloud.dev/blob` (https://github.com/google/go-cloud/tree/master/blob) for further simplicity.
 
-### Example
+## Proxied File system with HTTP Router
+
+You can store the files in using file system blob and can serve them using http file server. Upload library provides
+everything you need to do that in-built. Just use `pfsblob` as your file system handler. See the example
+in [pfsblob/tests](./pfsblob/tests).
+
+### URL format for pfsblob
+
+1. URL Scheme: `pfs`
+2. URL Host: Your domain name (without protocol) e.g. `localhost:8080` or `example.com`
+3. URL Path: Your storage directory in which you want to manage your files e.g. `/tmp/files`
+4. Domain Protocol: If your domain uses secure `https` protocol, set a query parameter: `?secure=true`
+5. Route: By default, files will be served at the root of the domain, if you want to change it, use query
+   parameter: `?route=static`
+
+Complete Query will look something like these:
+
+1. **`pfs://localhost:8080/tmp/files?secure=false`** <br/>
+   This will serve files at `http://localhost:8080/...` <br/>
+   e.g. the link for the file `/tmp/files/image.png` will be `http://localhost:8080/image.png`
+
+2. **`pfs://example.com/tmp/files?secure=true&route=public`** <br/>
+   This will serve files at `https://example.com/public/...` <br/>
+   e.g. the link for the file `/tmp/files/image.png` will be `https://example.com/public/image.png`
+
+## Example
+
 ```go
 package main
 
@@ -62,5 +91,6 @@ func main() {
 }
 ```
 
-### License
+## License
+
 This project is licensed under the [MIT License](./LICENSE)

bucket.go

@@ -6,7 +6,9 @@ import (
 	"fmt"
 	"io"
 	"net/url"
+	"strings"
 
+	pfs "github.com/Shivam010/upload/pfsblob"
 	"gocloud.dev/blob"
 	file "gocloud.dev/blob/fileblob"
 	gcs "gocloud.dev/blob/gcsblob"
@@ -22,13 +24,15 @@ const (
 	FileSystem
 	GoogleCloud
 	AmazonWebServices
+	ProxiedFileSystem
 )
 
 var ProviderName = map[Provider]string{
 	InMemory:          "In-Memory",
 	FileSystem:        "Local File System",
 	GoogleCloud:       "Google Cloud Console",
 	AmazonWebServices: "Amazon Web Services",
+	ProxiedFileSystem: "Proxied File System",
 }
 
 func (p Provider) String() string {
@@ -38,14 +42,14 @@ func (p Provider) String() string {
 type Bucket struct {
 	url      string
 	name     string
-	region   string
 	provider Provider
 	bucket   *blob.Bucket
+	metadata map[string]string
 }
 
 // NewBucket will return the blob bucket using the provided bucket url
 func NewBucket(bucket string) *Bucket {
-	b := &Bucket{url: bucket}
+	b := &Bucket{url: bucket, metadata: map[string]string{}}
 	b.parse()
 	return b
 }
@@ -85,7 +89,25 @@ func (b *Bucket) parse() {
 	case s3.Scheme:
 		b.provider = AmazonWebServices
 		b.name = u.Host
-		b.region = u.Query().Get("region")
+		b.metadata["region"] = u.Query().Get("region")
+	case pfs.Scheme:
+		b.provider = ProxiedFileSystem
+		// storage directory of the file system
+		b.metadata["storage"] = u.Path
+		// listening route region
+		route := strings.Trim(u.Query().Get("route"), "/")
+		b.metadata["route"] = route
+
+		// name of the bucket (for the link purposes)
+		b.name = "http://"
+		isSecure := u.Query().Get("secure")
+		if isSecure == "true" {
+			b.name = "https://"
+		}
+		b.name += u.Host
+		if route != "" {
+			b.name += "/" + route
+		}
 	default:
 		b.name = u.Host
 		b.url = mem.Scheme + "://" + b.name
@@ -98,11 +120,26 @@ func (b *Bucket) Provider() Provider {
 	return b.provider
 }
 
+// String implements stringer on Bucket
+func (b *Bucket) String() string {
+	return b.Provider().String() + " Bucket"
+}
+
 // Name returns name of the bucket
 func (b *Bucket) Name() string {
 	return b.name
 }
 
+// URL returns the url of the bucket
+func (b *Bucket) URL() string {
+	return b.url
+}
+
+// GetMetadata returns the value stored inside the key of the metadata of bucket
+func (b *Bucket) GetMetadata(key string) string {
+	return b.metadata[key]
+}
+
 // OpenContext opens a new bucket connection
 func (b *Bucket) OpenContext(ctx context.Context) (err error) {
 	b.parse()
@@ -156,7 +193,9 @@ func (b *Bucket) GetUrl(name string) string {
 	case GoogleCloud:
 		return fmt.Sprintf("https://storage.googleapis.com/%v/%v", b.name, name)
 	case AmazonWebServices:
-		return fmt.Sprintf("https://%v.s3.%v.amazonaws.com/%v", b.name, b.region, name)
+		return fmt.Sprintf("https://%v.s3.%v.amazonaws.com/%v", b.name, b.GetMetadata("region"), name)
+	case ProxiedFileSystem:
+		return b.name + "/" + name
 	}
 	return b.name + "/" + name
 }
@@ -172,7 +211,9 @@ func (b *Bucket) GetName(link string) string {
 	case GoogleCloud:
 		prefix = fmt.Sprintf("https://storage.googleapis.com/%v/", b.name)
 	case AmazonWebServices:
-		prefix = fmt.Sprintf("https://%v.s3.%v.amazonaws.com/", b.name, b.region)
+		prefix = fmt.Sprintf("https://%v.s3.%v.amazonaws.com/", b.name, b.GetMetadata("region"))
+	case ProxiedFileSystem:
+		prefix = b.name + "/"
 	}
 	if len(link) > len(prefix) {
 		return link[len(prefix):]

bucket_test.go

@@ -47,11 +47,11 @@ func GetTestData(t *testing.T) []unitData {
 			ioList: []struct{ input, output string }{
 				{
 					input:  "number/one.in",
-					output: getenv("S3_BASE_URL", "https://name.s3.us-east-2.amazonaws.com/")+"number/one.in",
+					output: getenv("S3_BASE_URL", "https://name.s3.us-east-2.amazonaws.com/") + "number/one.in",
 				},
 				{
 					input:  "two",
-					output: getenv("S3_BASE_URL", "https://name.s3.us-east-2.amazonaws.com/")+"two",
+					output: getenv("S3_BASE_URL", "https://name.s3.us-east-2.amazonaws.com/") + "two",
 				},
 			},
 		},
@@ -61,11 +61,53 @@ func GetTestData(t *testing.T) []unitData {
 			ioList: []struct{ input, output string }{
 				{
 					input:  "number/one.in",
-					output: getenv("GCP_BASE_URL", "https://storage.googleapis.com/name/")+"number/one.in",
+					output: getenv("GCP_BASE_URL", "https://storage.googleapis.com/name/") + "number/one.in",
 				},
 				{
 					input:  "two",
-					output: getenv("GCP_BASE_URL", "https://storage.googleapis.com/name/")+"two",
+					output: getenv("GCP_BASE_URL", "https://storage.googleapis.com/name/") + "two",
+				},
+			},
+		},
+		{
+			name:   ProxiedFileSystem.String(),
+			bucket: "pfs://localhost:8080" + pwd() + "/bin?route=web/file/srv",
+			ioList: []struct{ input, output string }{
+				{
+					input:  "number/three.in",
+					output: "http://localhost:8080/web/file/srv/number/three.in",
+				},
+				{
+					input:  "four",
+					output: "http://localhost:8080/web/file/srv/four",
+				},
+			},
+		},
+		{
+			name:   ProxiedFileSystem.String() + " (Secure)",
+			bucket: "pfs://example.com" + pwd() + "/bin/?route=web/file/srv&secure=true",
+			ioList: []struct{ input, output string }{
+				{
+					input:  "number/three.in",
+					output: "https://example.com/web/file/srv/number/three.in",
+				},
+				{
+					input:  "four",
+					output: "https://example.com/web/file/srv/four",
+				},
+			},
+		},
+		{
+			name:   ProxiedFileSystem.String() + " (without route)",
+			bucket: "pfs://example.com" + pwd() + "/bin",
+			ioList: []struct{ input, output string }{
+				{
+					input:  "number/three.in",
+					output: "http://example.com/number/three.in",
+				},
+				{
+					input:  "four",
+					output: "http://example.com/four",
 				},
 			},
 		},
@@ -85,10 +127,14 @@ func TestUpload(t *testing.T) {
 					continue
 				}
 				t.Log(bucket.Provider())
+				if bucket.Provider() == AmazonWebServices {
+					t.Skip("Credentials are removed")
+				}
 				// upload content
 				got, err := bucket.WriteAll(ctx, io.input, []byte(io.input))
 				if err != nil {
 					t.Errorf("WriteAll(%v), got: %v \n", io.input, err)
+					continue
 				}
 				if got != io.output {
 					t.Errorf("GetUrl(%v), got: %v want: %v \n", io.input, got, io.output)
@@ -97,6 +143,7 @@ func TestUpload(t *testing.T) {
 				con, err := bucket.ReadAll(ctx, bucket.GetName(io.output))
 				if err != nil {
 					t.Errorf("ReadAll(%v), got: %v \n", io.output, err)
+					continue
 				}
 				if string(con) != io.input {
 					t.Errorf("Content(%v), got: %s want: %v \n", io.output, con, io.input)
@@ -115,10 +162,14 @@ func TestUpload(t *testing.T) {
 			for _, io := range tt.ioList {
 				bucket := NewBucket(tt.bucket)
 				t.Log(bucket.Provider())
+				if bucket.Provider() == AmazonWebServices {
+					t.Skip("Credentials are removed")
+				}
 				// upload content
 				got, err := bucket.WriteAll(ctx, io.input, []byte(io.input))
 				if err != nil {
 					t.Errorf("WriteAll(%v), got: %v \n", io.input, err)
+					continue
 				}
 				if got != io.output {
 					t.Errorf("GetUrl(%v), got: %v want: %v \n", io.input, got, io.output)
@@ -127,14 +178,14 @@ func TestUpload(t *testing.T) {
 				r, err := bucket.Reader(ctx, bucket.GetName(io.output))
 				if err != nil {
 					t.Errorf("Reader(%v), got: %v \n", io.output, err)
+					continue
 				}
 				con, err := ioutil.ReadAll(r)
 				if err != nil {
 					t.Errorf("ReadAll(%v), got: %v \n", io.output, err)
 				}
 				if err := r.Close(); err != nil {
 					t.Errorf("Close(%v), got: %v \n", io.output, err)
-					return
 				}
 				if string(con) != io.input {
 					t.Errorf("Content(%v), got: %s want: %v \n", io.output, con, io.input)

go.mod

@@ -2,4 +2,7 @@ module github.com/Shivam010/upload
 
 go 1.16
 
-require gocloud.dev v0.23.0
+require (
+	gocloud.dev v0.23.0
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+)

go.sum

@@ -388,6 +388,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

pfsblob/handler/route.go

@@ -0,0 +1,53 @@
+package handler
+
+import (
+	"errors"
+	"github.com/Shivam010/upload"
+	"net/http"
+	"os"
+)
+
+func BucketRouteAndHandler(buck *upload.Bucket) (route string, handler func(http.ResponseWriter, *http.Request), err error) {
+	if buck == nil || buck.Provider() != upload.ProxiedFileSystem {
+		return "", nil, errors.New("handler: can only work on Proxied File System Bucket provider")
+	}
+	// Bucket region is alias for the route
+	// Bucket account is alias for the storage directory
+	route = "/" + buck.GetMetadata("route")
+	handlerFunc := http.StripPrefix(
+		route,
+		http.FileServer(wrappedFileSystem{
+			fs: http.Dir(buck.GetMetadata("storage")),
+		}),
+	).ServeHTTP
+
+	return route + "/",
+		func(w http.ResponseWriter, r *http.Request) {
+			// security and caching headers
+			w.Header().Set("X-Content-Type-Options", "nosniff")
+			w.Header().Set("Referrer-Policy", "strict-origin")
+			w.Header().Set("X-XSS-Protection", "1; mode=block")
+			w.Header().Set("X-Frame-Options", "SAMEORIGIN")
+			w.Header().Set("Strict-Transport-Security", "max-age=31536000")
+			w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
+			handlerFunc(w, r)
+		},
+		nil
+}
+
+type wrappedFileSystem struct {
+	fs http.FileSystem
+}
+
+func (w wrappedFileSystem) Open(path string) (http.File, error) {
+	file, err := w.fs.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	st, err := file.Stat()
+	if st.IsDir() {
+		return nil, os.ErrNotExist
+	}
+	return file, nil
+}