diff --git a/changelog/unreleased/uploads-refactor.md b/changelog/unreleased/uploads-refactor.md index f3ebb8e6e9..1bf82f9dbd 100644 --- a/changelog/unreleased/uploads-refactor.md +++ b/changelog/unreleased/uploads-refactor.md @@ -6,3 +6,4 @@ refactors that workflow to accept incoming requests following the tus protocol while using simpler transmission internally. https://github.com/cs3org/reva/pull/1285 +https://github.com/cs3org/reva/pull/1314 diff --git a/internal/http/services/owncloud/ocdav/get.go b/internal/http/services/owncloud/ocdav/get.go index 1517e3a803..98fb997892 100644 --- a/internal/http/services/owncloud/ocdav/get.go +++ b/internal/http/services/owncloud/ocdav/get.go @@ -142,7 +142,7 @@ func (s *svc) handleGet(w http.ResponseWriter, r *http.Request, ns string) { w.Header().Set("ETag", info.Etag) w.Header().Set("OC-FileId", wrapResourceID(info.Id)) w.Header().Set("OC-ETag", info.Etag) - t := utils.TSToTime(info.Mtime) + t := utils.TSToTime(info.Mtime).UTC() lastModifiedString := t.Format(time.RFC1123Z) w.Header().Set("Last-Modified", lastModifiedString) w.Header().Set("Content-Length", strconv.FormatUint(info.Size, 10)) diff --git a/internal/http/services/owncloud/ocdav/head.go b/internal/http/services/owncloud/ocdav/head.go index 02e776b560..d81452b577 100644 --- a/internal/http/services/owncloud/ocdav/head.go +++ b/internal/http/services/owncloud/ocdav/head.go @@ -76,7 +76,7 @@ func (s *svc) handleHead(w http.ResponseWriter, r *http.Request, ns string) { w.Header().Set("ETag", info.Etag) w.Header().Set("OC-FileId", wrapResourceID(info.Id)) w.Header().Set("OC-ETag", info.Etag) - t := utils.TSToTime(info.Mtime) + t := utils.TSToTime(info.Mtime).UTC() lastModifiedString := t.Format(time.RFC1123Z) w.Header().Set("Last-Modified", lastModifiedString) w.WriteHeader(http.StatusOK) diff --git a/internal/http/services/owncloud/ocdav/put.go b/internal/http/services/owncloud/ocdav/put.go index 61c4875b28..36de5aa544 100644 --- a/internal/http/services/owncloud/ocdav/put.go +++ b/internal/http/services/owncloud/ocdav/put.go @@ -312,7 +312,7 @@ func (s *svc) handlePutHelper(w http.ResponseWriter, r *http.Request, content io w.Header().Set("ETag", newInfo.Etag) w.Header().Set("OC-FileId", wrapResourceID(newInfo.Id)) w.Header().Set("OC-ETag", newInfo.Etag) - t := utils.TSToTime(newInfo.Mtime) + t := utils.TSToTime(newInfo.Mtime).UTC() lastModifiedString := t.Format(time.RFC1123Z) w.Header().Set("Last-Modified", lastModifiedString) diff --git a/internal/http/services/owncloud/ocdav/tus.go b/internal/http/services/owncloud/ocdav/tus.go index 0dd2490f2f..010d4c4f5b 100644 --- a/internal/http/services/owncloud/ocdav/tus.go +++ b/internal/http/services/owncloud/ocdav/tus.go @@ -264,7 +264,7 @@ func (s *svc) handleTusPost(w http.ResponseWriter, r *http.Request, ns string) { w.Header().Set("OC-FileId", wrapResourceID(info.Id)) w.Header().Set("OC-ETag", info.Etag) w.Header().Set("ETag", info.Etag) - t := utils.TSToTime(info.Mtime) + t := utils.TSToTime(info.Mtime).UTC() lastModifiedString := t.Format(time.RFC1123Z) w.Header().Set("Last-Modified", lastModifiedString) } diff --git a/pkg/storage/fs/ocis/upload.go b/pkg/storage/fs/ocis/upload.go index 3b8f0e9563..06830b5edf 100644 --- a/pkg/storage/fs/ocis/upload.go +++ b/pkg/storage/fs/ocis/upload.go @@ -44,7 +44,18 @@ var defaultFilePerm = os.FileMode(0664) func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) (err error) { upload, err := fs.GetUpload(ctx, ref.GetPath()) if err != nil { - return errors.Wrap(err, "ocisfs: error retrieving upload") + // Upload corresponding to this ID was not found. + // Assume that this corresponds to the resource path to which the file has to be uploaded. + + // Set the length to 0 and set SizeIsDeferred to true + metadata := map[string]string{"sizedeferred": "true"} + uploadID, err := fs.InitiateUpload(ctx, ref, 0, metadata) + if err != nil { + return err + } + if upload, err = fs.GetUpload(ctx, uploadID); err != nil { + return errors.Wrap(err, "ocisfs: error retrieving upload") + } } uploadInfo := upload.(*fileUpload) @@ -52,7 +63,7 @@ func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.Read p := uploadInfo.info.Storage["NodeName"] ok, err := chunking.IsChunked(p) if err != nil { - return errors.Wrap(err, "ocfs: error checking path") + return errors.Wrap(err, "ocisfs: error checking path") } if ok { var assembledFile string @@ -69,7 +80,7 @@ func (fs *ocisfs) Upload(ctx context.Context, ref *provider.Reference, r io.Read uploadInfo.info.Storage["NodeName"] = p fd, err := os.Open(assembledFile) if err != nil { - return errors.Wrap(err, "eos: error opening assembled file") + return errors.Wrap(err, "ocisfs: error opening assembled file") } defer fd.Close() defer os.RemoveAll(assembledFile) @@ -111,8 +122,13 @@ func (fs *ocisfs) InitiateUpload(ctx context.Context, ref *provider.Reference, u Size: uploadLength, } - if metadata != nil && metadata["mtime"] != "" { - info.MetaData["mtime"] = metadata["mtime"] + if metadata != nil { + if metadata["mtime"] != "" { + info.MetaData["mtime"] = metadata["mtime"] + } + if _, ok := metadata["sizedeferred"]; ok { + info.SizeIsDeferred = true + } } log.Debug().Interface("info", info).Interface("node", n).Interface("metadata", metadata).Msg("ocisfs: resolved filename") diff --git a/pkg/storage/fs/owncloud/upload.go b/pkg/storage/fs/owncloud/upload.go index d8e166ac42..a8e89f9442 100644 --- a/pkg/storage/fs/owncloud/upload.go +++ b/pkg/storage/fs/owncloud/upload.go @@ -44,7 +44,18 @@ var defaultFilePerm = os.FileMode(0664) func (fs *ocfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { upload, err := fs.GetUpload(ctx, ref.GetPath()) if err != nil { - return errors.Wrap(err, "ocfs: error retrieving upload") + // Upload corresponding to this ID was not found. + // Assume that this corresponds to the resource path to which the file has to be uploaded. + + // Set the length to 0 and set SizeIsDeferred to true + metadata := map[string]string{"sizedeferred": "true"} + uploadID, err := fs.InitiateUpload(ctx, ref, 0, metadata) + if err != nil { + return err + } + if upload, err = fs.GetUpload(ctx, uploadID); err != nil { + return errors.Wrap(err, "ocfs: error retrieving upload") + } } uploadInfo := upload.(*fileUpload) @@ -69,7 +80,7 @@ func (fs *ocfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCl uploadInfo.info.Storage["InternalDestination"] = p fd, err := os.Open(assembledFile) if err != nil { - return errors.Wrap(err, "eos: error opening assembled file") + return errors.Wrap(err, "ocfs: error opening assembled file") } defer fd.Close() defer os.RemoveAll(assembledFile) @@ -103,8 +114,13 @@ func (fs *ocfs) InitiateUpload(ctx context.Context, ref *provider.Reference, upl Size: uploadLength, } - if metadata != nil && metadata["mtime"] != "" { - info.MetaData["mtime"] = metadata["mtime"] + if metadata != nil { + if metadata["mtime"] != "" { + info.MetaData["mtime"] = metadata["mtime"] + } + if _, ok := metadata["sizedeferred"]; ok { + info.SizeIsDeferred = true + } } upload, err := fs.NewUpload(ctx, info) diff --git a/pkg/storage/utils/localfs/upload.go b/pkg/storage/utils/localfs/upload.go index 20fc1fa805..f749cb85aa 100644 --- a/pkg/storage/utils/localfs/upload.go +++ b/pkg/storage/utils/localfs/upload.go @@ -42,7 +42,18 @@ var defaultFilePerm = os.FileMode(0664) func (fs *localfs) Upload(ctx context.Context, ref *provider.Reference, r io.ReadCloser) error { upload, err := fs.GetUpload(ctx, ref.GetPath()) if err != nil { - return errors.Wrap(err, "ocisfs: error retrieving upload") + // Upload corresponding to this ID was not found. + // Assume that this corresponds to the resource path to which the file has to be uploaded. + + // Set the length to 0 and set SizeIsDeferred to true + metadata := map[string]string{"sizedeferred": "true"} + uploadID, err := fs.InitiateUpload(ctx, ref, 0, metadata) + if err != nil { + return err + } + if upload, err = fs.GetUpload(ctx, uploadID); err != nil { + return errors.Wrap(err, "localfs: error retrieving upload") + } } uploadInfo := upload.(*fileUpload) @@ -50,7 +61,7 @@ func (fs *localfs) Upload(ctx context.Context, ref *provider.Reference, r io.Rea p := uploadInfo.info.Storage["InternalDestination"] ok, err := chunking.IsChunked(p) if err != nil { - return errors.Wrap(err, "ocfs: error checking path") + return errors.Wrap(err, "localfs: error checking path") } if ok { var assembledFile string @@ -60,14 +71,14 @@ func (fs *localfs) Upload(ctx context.Context, ref *provider.Reference, r io.Rea } if p == "" { if err = uploadInfo.Terminate(ctx); err != nil { - return errors.Wrap(err, "ocfs: error removing auxiliary files") + return errors.Wrap(err, "localfs: error removing auxiliary files") } return errtypes.PartialContent(ref.String()) } uploadInfo.info.Storage["InternalDestination"] = p fd, err := os.Open(assembledFile) if err != nil { - return errors.Wrap(err, "eos: error opening assembled file") + return errors.Wrap(err, "localfs: error opening assembled file") } defer fd.Close() defer os.RemoveAll(assembledFile) @@ -75,7 +86,7 @@ func (fs *localfs) Upload(ctx context.Context, ref *provider.Reference, r io.Rea } if _, err := uploadInfo.WriteChunk(ctx, 0, r); err != nil { - return errors.Wrap(err, "ocisfs: error writing to binary file") + return errors.Wrap(err, "localfs: error writing to binary file") } return uploadInfo.FinishUpload(ctx) @@ -101,8 +112,13 @@ func (fs *localfs) InitiateUpload(ctx context.Context, ref *provider.Reference, Size: uploadLength, } - if metadata != nil && metadata["mtime"] != "" { - info.MetaData["mtime"] = metadata["mtime"] + if metadata != nil { + if metadata["mtime"] != "" { + info.MetaData["mtime"] = metadata["mtime"] + } + if _, ok := metadata["sizedeferred"]; ok { + info.SizeIsDeferred = true + } } upload, err := fs.NewUpload(ctx, info) @@ -331,8 +347,10 @@ func (upload *fileUpload) FinishUpload(ctx context.Context) error { // only delete the upload if it was successfully written to the fs if err := os.Remove(upload.infoPath); err != nil { - log := appctx.GetLogger(ctx) - log.Err(err).Interface("info", upload.info).Msg("localfs: could not delete upload info") + if !os.IsNotExist(err) { + log := appctx.GetLogger(ctx) + log.Err(err).Interface("info", upload.info).Msg("localfs: could not delete upload info") + } } // TODO: set mtime if specified in metadata