diff --git a/backend/posix/posix.go b/backend/posix/posix.go index f437de18..67532f0b 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -86,6 +86,9 @@ const ( objectRetentionKey = "object-retention" objectLegalHoldKey = "object-legal-hold" + // this is the media type for directories in AWS and Nextcloud + dirContentType = "application/x-directory" + doFalloc = true skipFalloc = false ) @@ -474,7 +477,8 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipa } // set content-type - if *mpu.ContentType != "" { + ctype := getString(mpu.ContentType) + if ctype != "" { err := p.meta.StoreAttribute(bucket, filepath.Join(objdir, uploadID), contentTypeHdr, []byte(*mpu.ContentType)) if err != nil { @@ -485,6 +489,19 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipa } } + // set content-encoding + cenc := getString(mpu.ContentEncoding) + if cenc != "" { + err := p.meta.StoreAttribute(bucket, filepath.Join(objdir, uploadID), contentEncHdr, + []byte(*mpu.ContentEncoding)) + if err != nil { + // cleanup object if returning error + os.RemoveAll(filepath.Join(tmppath, uploadID)) + os.Remove(tmppath) + return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("set content-encoding: %w", err) + } + } + // set object legal hold if mpu.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { if err := p.PutObjectLegalHold(ctx, bucket, filepath.Join(objdir, uploadID), "", true); err != nil { @@ -649,7 +666,7 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM userMetaData := make(map[string]string) upiddir := filepath.Join(objdir, uploadID) - cType, _ := p.loadUserMetaData(bucket, upiddir, userMetaData) + cType, cEnc := p.loadUserMetaData(bucket, upiddir, userMetaData) objname := filepath.Join(bucket, object) dir := filepath.Dir(objname) @@ -696,6 +713,15 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM } } + // set content-encoding + if cEnc != "" { + if err := p.meta.StoreAttribute(bucket, object, contentEncHdr, []byte(cEnc)); err != nil { + // cleanup object + os.Remove(objname) + return nil, fmt.Errorf("set object content encoding: %w", err) + } + } + // load and set legal hold lHold, err := p.meta.RetrieveAttribute(bucket, upiddir, objectLegalHoldKey) if err == nil { @@ -796,15 +822,9 @@ func (p *Posix) loadUserMetaData(bucket, object string, m map[string]string) (st var contentType, contentEncoding string b, _ := p.meta.RetrieveAttribute(bucket, object, contentTypeHdr) contentType = string(b) - if contentType != "" { - m[contentTypeHdr] = contentType - } b, _ = p.meta.RetrieveAttribute(bucket, object, contentEncHdr) contentEncoding = string(b) - if contentEncoding != "" { - m[contentEncHdr] = contentEncoding - } return contentType, contentEncoding } @@ -1408,7 +1428,8 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e } // set etag attribute to signify this dir was specifically put - err = p.meta.StoreAttribute(*po.Bucket, *po.Key, etagkey, []byte(emptyMD5)) + err = p.meta.StoreAttribute(*po.Bucket, *po.Key, etagkey, + []byte(emptyMD5)) if err != nil { return "", fmt.Errorf("set etag attr: %w", err) } @@ -1478,7 +1499,8 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e // Set object legal hold if po.ObjectLockLegalHoldStatus == types.ObjectLockLegalHoldStatusOn { - if err := p.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true); err != nil { + err := p.PutObjectLegalHold(ctx, *po.Bucket, *po.Key, "", true) + if err != nil { return "", err } } @@ -1493,7 +1515,8 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e if err != nil { return "", fmt.Errorf("parse object lock retention: %w", err) } - if err := p.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed); err != nil { + err = p.PutObjectRetention(ctx, *po.Bucket, *po.Key, "", true, retParsed) + if err != nil { return "", err } } @@ -1505,6 +1528,24 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (string, e return "", fmt.Errorf("set etag attr: %w", err) } + ctype := getString(po.ContentType) + if ctype != "" { + err := p.meta.StoreAttribute(*po.Bucket, *po.Key, contentTypeHdr, + []byte(*po.ContentType)) + if err != nil { + return "", fmt.Errorf("set content-type attr: %w", err) + } + } + + cenc := getString(po.ContentEncoding) + if cenc != "" { + err := p.meta.StoreAttribute(*po.Bucket, *po.Key, contentEncHdr, + []byte(*po.ContentEncoding)) + if err != nil { + return "", fmt.Errorf("set content-encoding attr: %w", err) + } + } + return etag, nil } @@ -1697,7 +1738,8 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO if fi.IsDir() { userMetaData := make(map[string]string) - contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData) + _, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData) + contentType := dirContentType b, err := p.meta.RetrieveAttribute(bucket, object, etagkey) etag := string(b) @@ -1856,8 +1898,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. contentType, contentEncoding := p.loadUserMetaData(bucket, object, userMetaData) if fi.IsDir() { - // this is the media type for directories in AWS and Nextcloud - contentType = "application/x-directory" + contentType = dirContentType } b, err := p.meta.RetrieveAttribute(bucket, object, etagkey) diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 9415aab9..93438dde 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -1530,6 +1530,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { versionId := ctx.Query("versionId") acct := ctx.Locals("account").(auth.Account) isRoot := ctx.Locals("isRoot").(bool) + contentType := ctx.Get("Content-Type") + contentEncoding := ctx.Get("Content-Encoding") parsedAcl := ctx.Locals("parsedAcl").(auth.ACL) tagging := ctx.Get("x-amz-tagging") @@ -2235,6 +2237,8 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Bucket: &bucket, Key: &keyStart, ContentLength: &contentLength, + ContentType: &contentType, + ContentEncoding: &contentEncoding, Metadata: metadata, Body: body, Tagging: &tagging, @@ -2842,6 +2846,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { isRoot := ctx.Locals("isRoot").(bool) parsedAcl := ctx.Locals("parsedAcl").(auth.ACL) contentType := ctx.Get("Content-Type") + contentEncoding := ctx.Get("Content-Encoding") tagging := ctx.Get("X-Amz-Tagging") if keyEnd != "" { @@ -3071,6 +3076,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { Key: &key, Tagging: &tagging, ContentType: &contentType, + ContentEncoding: &contentEncoding, ObjectLockRetainUntilDate: &objLockState.RetainUntilDate, ObjectLockMode: objLockState.ObjectLockMode, ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus, diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index c99dd7c4..fee493f6 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -145,6 +145,7 @@ func TestHeadObject(s *S3Conf) { HeadObject_mp_success(s) HeadObject_non_existing_dir_object(s) HeadObject_name_too_long(s) + HeadObject_with_contenttype(s) HeadObject_success(s) } @@ -161,6 +162,7 @@ func TestGetObject(s *S3Conf) { GetObject_invalid_ranges(s) GetObject_with_meta(s) GetObject_success(s) + GetObject_directory_success(s) GetObject_by_range_success(s) GetObject_by_range_resp_status(s) GetObject_non_existing_dir_object(s) @@ -596,6 +598,7 @@ func GetIntTests() IntTests { "HeadObject_mp_success": HeadObject_mp_success, "HeadObject_non_existing_dir_object": HeadObject_non_existing_dir_object, "HeadObject_name_too_long": HeadObject_name_too_long, + "HeadObject_with_contenttype": HeadObject_with_contenttype, "HeadObject_success": HeadObject_success, "GetObjectAttributes_non_existing_bucket": GetObjectAttributes_non_existing_bucket, "GetObjectAttributes_non_existing_object": GetObjectAttributes_non_existing_object, @@ -606,6 +609,7 @@ func GetIntTests() IntTests { "GetObject_invalid_ranges": GetObject_invalid_ranges, "GetObject_with_meta": GetObject_with_meta, "GetObject_success": GetObject_success, + "GetObject_directory_success": GetObject_directory_success, "GetObject_by_range_success": GetObject_by_range_success, "GetObject_by_range_resp_status": GetObject_by_range_resp_status, "GetObject_non_existing_dir_object": GetObject_non_existing_dir_object, diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 7acca091..26fb9e5e 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -3028,6 +3028,60 @@ func HeadObject_non_existing_dir_object(s *S3Conf) error { const defaultContentType = "binary/octet-stream" +func HeadObject_with_contenttype(s *S3Conf) error { + testName := "HeadObject_with_contenttype" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj, dataLen := "my-obj", int64(1234567) + contentType := "text/plain" + contentEncoding := "gzip" + + _, _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + ContentType: &contentType, + ContentEncoding: &contentEncoding, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + + contentLength := int64(0) + if out.ContentLength != nil { + contentLength = *out.ContentLength + } + if contentLength != dataLen { + return fmt.Errorf("expected data length %v, instead got %v", dataLen, contentLength) + } + if out.ContentType == nil { + return fmt.Errorf("expected content type %v, instead got nil", contentType) + } + if *out.ContentType != contentType { + return fmt.Errorf("expected content type %v, instead got %v", contentType, *out.ContentType) + } + if out.ContentEncoding == nil { + return fmt.Errorf("expected content encoding %v, instead got nil", contentEncoding) + } + if *out.ContentEncoding != contentEncoding { + return fmt.Errorf("expected content encoding %v, instead got %v", contentEncoding, *out.ContentEncoding) + } + if out.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) + } + + return nil + }) +} + func HeadObject_success(s *S3Conf) error { testName := "HeadObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -3036,11 +3090,13 @@ func HeadObject_success(s *S3Conf) error { "key1": "val1", "key2": "val2", } + ctype := defaultContentType _, _, err := putObjectWithData(dataLen, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - Metadata: meta, + Bucket: &bucket, + Key: &obj, + Metadata: meta, + ContentType: &ctype, }, s3client) if err != nil { return err @@ -3443,10 +3499,12 @@ func GetObject_success(s *S3Conf) error { testName := "GetObject_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { dataLength, obj := int64(1234567), "my-obj" + ctype := defaultContentType csum, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, + Bucket: &bucket, + Key: &obj, + ContentType: &ctype, }, s3client) if err != nil { return err @@ -3484,6 +3542,45 @@ func GetObject_success(s *S3Conf) error { }) } +const directoryContentType = "application/x-directory" + +func GetObject_directory_success(s *S3Conf) error { + testName := "GetObject_directory_success" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + dataLength, obj := int64(0), "my-dir/" + + _, _, err := putObjectWithData(dataLength, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &obj, + }, s3client) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &obj, + }) + defer cancel() + if err != nil { + return err + } + if *out.ContentLength != dataLength { + return fmt.Errorf("expected content-length %v, instead got %v", dataLength, out.ContentLength) + } + if *out.ContentType != directoryContentType { + return fmt.Errorf("expected content type %v, instead got %v", directoryContentType, *out.ContentType) + } + if out.StorageClass != types.StorageClassStandard { + return fmt.Errorf("expected the storage class to be %v, instead got %v", types.StorageClassStandard, out.StorageClass) + } + + out.Body.Close() + return nil + }) +} + func GetObject_by_range_success(s *S3Conf) error { testName := "GetObject_by_range_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -5087,11 +5184,16 @@ func CreateMultipartUpload_with_metadata(s *S3Conf) error { "prop1": "val1", "prop2": "val2", } + contentType := "application/text" + contentEncoding := "testenc" + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ - Bucket: &bucket, - Key: &obj, - Metadata: meta, + Bucket: &bucket, + Key: &obj, + Metadata: meta, + ContentType: &contentType, + ContentEncoding: &contentEncoding, }) cancel() if err != nil { @@ -5139,6 +5241,19 @@ func CreateMultipartUpload_with_metadata(s *S3Conf) error { return fmt.Errorf("expected uploaded object metadata to be %v, instead got %v", meta, resp.Metadata) } + if resp.ContentType == nil { + return fmt.Errorf("expected uploaded object content-type to be %v, instead got nil", contentType) + } + if *resp.ContentType != contentType { + return fmt.Errorf("expected uploaded object content-type to be %v, instead got %v", contentType, *resp.ContentType) + } + if resp.ContentEncoding == nil { + return fmt.Errorf("expected uploaded object content-encoding to be %v, instead got nil", contentEncoding) + } + if *resp.ContentEncoding != contentEncoding { + return fmt.Errorf("expected uploaded object content-encoding to be %v, instead got %v", contentEncoding, *resp.ContentEncoding) + } + return nil }) }