From 7107b85013300c2d9d3ca48d384225620020dc26 Mon Sep 17 00:00:00 2001 From: Vincent Petry Date: Thu, 28 May 2020 12:09:43 +0200 Subject: [PATCH 1/3] Attempt to return metadata after a TUS PATCH --- .../services/dataprovider/dataprovider.go | 35 +++++++++++++++++++ internal/http/services/dataprovider/put.go | 2 ++ 2 files changed, 37 insertions(+) diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index ff72e11fa0..9b369fab6c 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -21,7 +21,11 @@ package dataprovider import ( "fmt" "net/http" + "strings" + "time" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/internal/http/utils" "github.com/cs3org/reva/pkg/appctx" "github.com/cs3org/reva/pkg/rhttp/global" "github.com/cs3org/reva/pkg/storage" @@ -107,6 +111,34 @@ type Composable interface { UseIn(composer *tusd.StoreComposer) } +func (s *svc) writeFileInfoHeaders(w http.ResponseWriter, r *http.Request) error { + ctx := r.Context() + log := appctx.GetLogger(ctx) + log.Debug().Msg("dataprovider: writeFileInfoHeaders()") + fn := r.URL.Path + + fsfn := strings.TrimPrefix(fn, s.conf.Prefix) + //ref := &provider.Reference{Spec: &provider.Reference_Path{Path: fsfn}} + // TODO: try reading info data + ref := &provider.Reference{Spec: &provider.Reference_Path{Path: fsfn}} + sRes, err := s.storage.GetMD(ctx, ref) + if err != nil { + log.Error().Err(err).Msg("error sending grpc GetMD request after upload") + return err + } + + w.Header().Add("Content-Type", sRes.GetMimeType()) + w.Header().Set("ETag", sRes.GetEtag()) + // FIXME: implement wrap on this layer + //w.Header().Set("OC-FileId", wrapResourceID(sRes.GetId())) + w.Header().Set("OC-ETag", sRes.GetEtag()) + t := utils.TSToTime(sRes.GetMtime()) + lastModifiedString := t.Format(time.RFC1123) + w.Header().Set("Last-Modified", lastModifiedString) + w.Header().Set("X-OC-MTime", "accepted") + return nil +} + func (s *svc) setHandler() (err error) { composable, ok := s.storage.(Composable) if ok && !s.conf.DisableTus { @@ -159,10 +191,13 @@ func (s *svc) setHandler() (err error) { handler.HeadFile(w, r) case "PATCH": handler.PatchFile(w, r) + // FIXME: we can't set additional headers on the already sent response + s.writeFileInfoHeaders(w, r) // PUT provides a wrapper around the POST call, to save the caller from // the trouble of configuring the tus client. case "PUT": s.doTusPut(w, r) + s.writeFileInfoHeaders(w, r) // TODO Only attach the DELETE handler if the Terminate() method is provided case "DELETE": handler.DelFile(w, r) diff --git a/internal/http/services/dataprovider/put.go b/internal/http/services/dataprovider/put.go index 6471b475d5..8783cb982b 100644 --- a/internal/http/services/dataprovider/put.go +++ b/internal/http/services/dataprovider/put.go @@ -36,6 +36,7 @@ import ( func (s *svc) doPut(w http.ResponseWriter, r *http.Request) { ctx := r.Context() log := appctx.GetLogger(ctx) + log.Debug().Msg("dataprovider: doPut()") fn := r.URL.Path fsfn := strings.TrimPrefix(fn, s.conf.Prefix) @@ -49,6 +50,7 @@ func (s *svc) doPut(w http.ResponseWriter, r *http.Request) { } r.Body.Close() + w.WriteHeader(http.StatusOK) } From b9ba741f81156f7f7605af4cb4dd2db1ec605af2 Mon Sep 17 00:00:00 2001 From: Vincent Petry Date: Tue, 9 Jun 2020 15:47:31 +0200 Subject: [PATCH 2/3] Use response wrapper to inject headers after TUS handler --- .../services/dataprovider/dataprovider.go | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index 9b369fab6c..555a49273a 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -52,6 +52,41 @@ type svc struct { storage storage.FS } +type WrappedTusHandler struct { + tusd.UnroutedHandler +} + +func (handler *WrappedTusHandler) sendResp(w http.ResponseWriter, r *http.Request, status int) { + // TODO: inject extra response headers + w.Header().Add("X-Override-Hack", "WrappedTusHandler") + w.WriteHeader(status) + + // FIXME: how to call parent log ? + //handler.UnroutedHandler.log("ResponseOutgoing", "status", strconv.Itoa(status), "method", r.Method, "path", r.URL.Path, "requestId", getRequestId(r)) +} + +type WrappedResponseWriter struct { + http.ResponseWriter + statusCode int +} + +func (w *WrappedResponseWriter) WriteHeader(statusCode int) { + w.statusCode = statusCode + // delay this +} + +func (w *WrappedResponseWriter) SendResponse() { + w.ResponseWriter.WriteHeader(w.statusCode) +} + +func (w *WrappedResponseWriter) Header() http.Header { + return w.ResponseWriter.Header() +} + +func (w *WrappedResponseWriter) Write(bytes []byte) (int, error) { + return w.ResponseWriter.Write(bytes) +} + // New returns a new datasvc func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) { conf := &config{} @@ -190,9 +225,14 @@ func (s *svc) setHandler() (err error) { case "HEAD": handler.HeadFile(w, r) case "PATCH": - handler.PatchFile(w, r) - // FIXME: we can't set additional headers on the already sent response - s.writeFileInfoHeaders(w, r) + // HACK: make it possible to send headers after the TUS handler has already sent the response + wrappedWriter := &WrappedResponseWriter{ResponseWriter: w} + handler.PatchFile(wrappedWriter, r) + + //s.writeFileInfoHeaders(w, r) + //if w.Header().Get("Upload-Offset") == w.Header().Get("Upload-Length") + w.Header().Add("X-Test", "123") + wrappedWriter.SendResponse() // PUT provides a wrapper around the POST call, to save the caller from // the trouble of configuring the tus client. case "PUT": From c19f495641b74951761753cdde46b28168e5005c Mon Sep 17 00:00:00 2001 From: Vincent Petry Date: Tue, 9 Jun 2020 16:10:14 +0200 Subject: [PATCH 3/3] Send metadata headers on upload completion --- .../services/dataprovider/dataprovider.go | 54 +++++++++++-------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/internal/http/services/dataprovider/dataprovider.go b/internal/http/services/dataprovider/dataprovider.go index 555a49273a..af2926a0c1 100644 --- a/internal/http/services/dataprovider/dataprovider.go +++ b/internal/http/services/dataprovider/dataprovider.go @@ -52,19 +52,6 @@ type svc struct { storage storage.FS } -type WrappedTusHandler struct { - tusd.UnroutedHandler -} - -func (handler *WrappedTusHandler) sendResp(w http.ResponseWriter, r *http.Request, status int) { - // TODO: inject extra response headers - w.Header().Add("X-Override-Hack", "WrappedTusHandler") - w.WriteHeader(status) - - // FIXME: how to call parent log ? - //handler.UnroutedHandler.log("ResponseOutgoing", "status", strconv.Itoa(status), "method", r.Method, "path", r.URL.Path, "requestId", getRequestId(r)) -} - type WrappedResponseWriter struct { http.ResponseWriter statusCode int @@ -146,13 +133,15 @@ type Composable interface { UseIn(composer *tusd.StoreComposer) } -func (s *svc) writeFileInfoHeaders(w http.ResponseWriter, r *http.Request) error { +func (s *svc) writeFileInfoHeaders(w http.ResponseWriter, r *http.Request, dest string) error { ctx := r.Context() log := appctx.GetLogger(ctx) log.Debug().Msg("dataprovider: writeFileInfoHeaders()") - fn := r.URL.Path + fn := dest + // FIXME: seems trim prefix doesn't trim anything fsfn := strings.TrimPrefix(fn, s.conf.Prefix) + log.Debug().Str("dest", dest).Str("fsfn", fsfn).Msg("writeFileInfoHeaders") //ref := &provider.Reference{Spec: &provider.Reference_Path{Path: fsfn}} // TODO: try reading info data ref := &provider.Reference{Spec: &provider.Reference_Path{Path: fsfn}} @@ -162,6 +151,7 @@ func (s *svc) writeFileInfoHeaders(w http.ResponseWriter, r *http.Request) error return err } + log.Debug().Interface("fileid", sRes.GetId()).Msg("Got fileid") w.Header().Add("Content-Type", sRes.GetMimeType()) w.Header().Set("ETag", sRes.GetEtag()) // FIXME: implement wrap on this layer @@ -187,8 +177,9 @@ func (s *svc) setHandler() (err error) { composable.UseIn(composer) config := tusd.Config{ - BasePath: s.conf.Prefix, - StoreComposer: composer, + BasePath: s.conf.Prefix, + StoreComposer: composer, + NotifyCompleteUploads: true, //Logger: logger, // TODO use logger } @@ -227,17 +218,38 @@ func (s *svc) setHandler() (err error) { case "PATCH": // HACK: make it possible to send headers after the TUS handler has already sent the response wrappedWriter := &WrappedResponseWriter{ResponseWriter: w} + + var uploadStorage map[string]string + + // HACK: need to get access to the upload info, which is only accessible + // through the events + quit := make(chan struct{}) + go func() { + for { + select { + case info := <-handler.CompleteUploads: + if info.HTTPRequest.URI == r.RequestURI { + uploadStorage = info.Upload.Storage + } + case <-quit: + return + } + } + }() + handler.PatchFile(wrappedWriter, r) + close(quit) + + if uploadStorage != nil { + log.Debug().Interface("uploadStorage", uploadStorage).Msg("Upload complete") + s.writeFileInfoHeaders(w, r, uploadStorage["InternalDestination"]) + } - //s.writeFileInfoHeaders(w, r) - //if w.Header().Get("Upload-Offset") == w.Header().Get("Upload-Length") - w.Header().Add("X-Test", "123") wrappedWriter.SendResponse() // PUT provides a wrapper around the POST call, to save the caller from // the trouble of configuring the tus client. case "PUT": s.doTusPut(w, r) - s.writeFileInfoHeaders(w, r) // TODO Only attach the DELETE handler if the Terminate() method is provided case "DELETE": handler.DelFile(w, r)