Skip to content

Commit

Permalink
Compose Obejct method (#850)
Browse files Browse the repository at this point in the history 
* Compose Obejct method

* fixing comments

* fixing comments

* addding more testcase

* fixing comments

* fixing comments

* fixing lint test

* fixing lint test

* fixing lint test

* fixing lint test

* fixing lint test

* fixing comments

* fixing comments

* fixing comments

* fixing comments

* fixing comments

* fixing comments

* fixing lint test

* adding one condition for srcobject does not exist

* adding one condition for srcobject does not exist

* fixing comment

* fixing comment

* fixing comments

* fixing lint test

* fixing lint test

* fixing lint test

* fixing lint test

* fixing lint test

* fixing comments

* adding private method for read

* fixing generation condition

* fixing lint test

* fixing lint test

* removing unnecessary changes

* Adding Name Method

* Adding Unit Test

* Adding Unit Test for storage handle

* Adding comment

* Fixing Comments

* Fixing Comments

* Fixing Comments

* Fixing conflicts

* Fixing Conflicts

* Fixing Conflicts

* Formating file
  • Loading branch information
@Tulsishah
Tulsishah authored Nov 29, 2022
1 parent e24ecc9 commit 74689e0
Show file tree
Hide file tree
Showing 3 changed files with 341 additions and 1 deletion.
46 changes: 46 additions & 0 deletions internal/storage/bucket_handle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,3 +316,49 @@ func (b *bucketHandle) UpdateObject(ctx context.Context, req *gcs.UpdateObjectRe

return
}

func (b *bucketHandle) ComposeObjects(ctx context.Context, req *gcs.ComposeObjectsRequest) (o *gcs.Object, err error) {
dstObj := b.bucket.Object(req.DstName)

if req.DstGenerationPrecondition != nil && req.DstMetaGenerationPrecondition != nil {
dstObj = dstObj.If(storage.Conditions{GenerationMatch: *req.DstGenerationPrecondition, MetagenerationMatch: *req.DstMetaGenerationPrecondition})
} else if req.DstGenerationPrecondition != nil {
dstObj = dstObj.If(storage.Conditions{GenerationMatch: *req.DstGenerationPrecondition})
} else if req.DstMetaGenerationPrecondition != nil {
dstObj = dstObj.If(storage.Conditions{MetagenerationMatch: *req.DstMetaGenerationPrecondition})
}

// Converting the req.Sources list to a list of storage.ObjectHandle as expected by the Go Storage Client.
var srcObjList []*storage.ObjectHandle
for _, src := range req.Sources {
currSrcObj := b.bucket.Object(src.Name)
// Switching to requested Generation of the object.
// Zero src generation is the latest generation, we are skipping it because by default it will take the latest one
if src.Generation != 0 {
currSrcObj = currSrcObj.Generation(src.Generation)
}
srcObjList = append(srcObjList, currSrcObj)
}

// Composing Source Objects to Destination Object using Composer created through Go Storage Client.
attrs, err := dstObj.ComposerFrom(srcObjList...).Run(ctx)
if err != nil {
switch ee := err.(type) {
case *googleapi.Error:
if ee.Code == http.StatusPreconditionFailed {
err = &gcs.PreconditionError{Err: ee}
}
if ee.Code == http.StatusNotFound {
err = &gcs.NotFoundError{Err: storage.ErrObjectNotExist}
}
default:
err = fmt.Errorf("Error in composing object: %w", err)
}
return
}

// Converting attrs to type *Object.
o = storageutil.ObjectAttrsToBucketObject(attrs)

return
}
293 changes: 293 additions & 0 deletions internal/storage/bucket_handle_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ var ContentType string = "ContentType"
var ContentEncoding string = "ContentEncoding"
var ContentLanguage string = "ContentLanguage"
var CacheControl string = "CacheControl"
var CustomTime string = "CustomTime"
var StorageClass string = "StorageClass"
var ContentDisposition string = "ContentDisposition"

// FakeGCSServer is not handling generation and metageneration checks for Delete flow.
// Hence, we are not writing tests for these flows.
Expand Down Expand Up @@ -468,6 +471,296 @@ func (t *BucketHandleTest) TestUpdateObjectMethodWithMissingObject() {
AssertTrue(errors.As(err, &notfound))
}

// Read content of an object and return
func (t *BucketHandleTest) readObjectContent(ctx context.Context, req *gcs.ReadObjectRequest) (buffer string) {
rc, err := t.bucketHandle.NewReader(ctx, &gcs.ReadObjectRequest{
Name: req.Name,
Range: req.Range})

AssertEq(nil, err)
defer rc.Close()
buf := make([]byte, req.Range.Limit)
_, err = rc.Read(buf)
AssertEq(nil, err)
return string(buf[:])
}

func (t *BucketHandleTest) TestComposeObjectMethodWithDstObjectExist() {
// Reading content before composing it
buffer := t.readObjectContent(context.Background(),
&gcs.ReadObjectRequest{
Name: TestObjectName,
Range: &gcs.ByteRange{
Start: uint64(0),
Limit: uint64(len(ContentInTestObject)),
},
})
ExpectEq(ContentInTestObject, buffer)
// Checking if srcObject exists or not
srcObj, err := t.bucketHandle.StatObject(context.Background(),
&gcs.StatObjectRequest{
Name: TestSubObjectName,
})
AssertEq(nil, err)
AssertNe(nil, srcObj)

// Composing the object
composedObj, err := t.bucketHandle.ComposeObjects(context.Background(),
&gcs.ComposeObjectsRequest{
DstName: TestObjectName,
DstGenerationPrecondition: nil,
DstMetaGenerationPrecondition: nil,
Sources: []gcs.ComposeSource{
{
Name: TestSubObjectName,
},
},
ContentType: ContentType,
Metadata: map[string]string{
MetaDataKey: MetaDataValue,
},
ContentLanguage: ContentLanguage,
ContentEncoding: ContentEncoding,
CacheControl: CacheControl,
ContentDisposition: ContentDisposition,
CustomTime: CustomTime,
EventBasedHold: true,
StorageClass: StorageClass,
Acl: nil,
})

AssertEq(nil, err)
// Validation of srcObject to ensure that it is not effected.
srcBuffer := t.readObjectContent(context.Background(),
&gcs.ReadObjectRequest{
Name: TestSubObjectName,
Range: &gcs.ByteRange{
Start: uint64(0),
Limit: uint64(len(ContentInTestSubObject)),
},
})
ExpectEq(ContentInTestSubObject, srcBuffer)
// Reading content of destination object
dstBuffer := t.readObjectContent(context.Background(),
&gcs.ReadObjectRequest{
Name: TestObjectName,
Range: &gcs.ByteRange{
Start: uint64(0),
Limit: uint64(composedObj.Size),
},
})
// Destination object's content will get overwrite by srcObject.
ExpectEq(srcBuffer, dstBuffer)
AssertNe(nil, composedObj)
AssertEq(srcObj.Size, composedObj.Size)
}

func (t *BucketHandleTest) TestComposeObjectMethodWithOneSrcObject() {
var notfound *gcs.NotFoundError
// Checking that dstObject does not exist
_, err := t.bucketHandle.StatObject(context.Background(),
&gcs.StatObjectRequest{
Name: dstObjectName,
})
AssertTrue(errors.As(err, &notfound))
srcObj, err := t.bucketHandle.StatObject(context.Background(),
&gcs.StatObjectRequest{
Name: TestObjectName,
})
AssertEq(nil, err)
AssertNe(nil, srcObj)

composedObj, err := t.bucketHandle.ComposeObjects(context.Background(),
&gcs.ComposeObjectsRequest{
DstName: dstObjectName,
DstGenerationPrecondition: nil,
DstMetaGenerationPrecondition: nil,
Sources: []gcs.ComposeSource{
{
Name: TestObjectName,
},
},
ContentType: ContentType,
Metadata: map[string]string{
MetaDataKey: MetaDataValue,
},
ContentLanguage: ContentLanguage,
ContentEncoding: ContentEncoding,
CacheControl: CacheControl,
ContentDisposition: ContentDisposition,
CustomTime: CustomTime,
EventBasedHold: true,
StorageClass: StorageClass,
Acl: nil,
})

AssertEq(nil, err)
// Validation of srcObject to ensure that it is not effected.
srcBuffer := t.readObjectContent(context.Background(),
&gcs.ReadObjectRequest{
Name: TestObjectName,
Range: &gcs.ByteRange{
Start: uint64(0),
Limit: uint64(composedObj.Size),
},
})
// Reading content of dstObject
dstBuffer := t.readObjectContent(context.Background(),
&gcs.ReadObjectRequest{
Name: dstObjectName,
Range: &gcs.ByteRange{
Start: uint64(0),
Limit: uint64(composedObj.Size),
},
})
ExpectEq(srcBuffer, dstBuffer)
AssertNe(nil, composedObj)
AssertEq(srcObj.Size, composedObj.Size)
}

func (t *BucketHandleTest) TestComposeObjectMethodWithTwoSrcObjects() {
var notfound *gcs.NotFoundError
_, err := t.bucketHandle.StatObject(context.Background(),
&gcs.StatObjectRequest{
Name: dstObjectName,
})
AssertTrue(errors.As(err, &notfound))
srcObj1, err := t.bucketHandle.StatObject(context.Background(),
&gcs.StatObjectRequest{
Name: TestObjectName,
})
AssertEq(nil, err)
AssertNe(nil, srcObj1)
srcObj2, err := t.bucketHandle.StatObject(context.Background(),
&gcs.StatObjectRequest{
Name: TestSubObjectName,
})
AssertEq(nil, err)
AssertNe(nil, srcObj2)

composedObj, err := t.bucketHandle.ComposeObjects(context.Background(),
&gcs.ComposeObjectsRequest{
DstName: dstObjectName,
DstGenerationPrecondition: nil,
DstMetaGenerationPrecondition: nil,
Sources: []gcs.ComposeSource{
{
Name: TestObjectName,
},
{
Name: TestSubObjectName,
},
},
ContentType: ContentType,
Metadata: map[string]string{
MetaDataKey: MetaDataValue,
},
ContentLanguage: ContentLanguage,
ContentEncoding: ContentEncoding,
CacheControl: CacheControl,
ContentDisposition: ContentDisposition,
CustomTime: CustomTime,
EventBasedHold: true,
StorageClass: StorageClass,
Acl: nil,
})

AssertEq(nil, err)
// Validation of srcObject1 to ensure that it is not effected.
srcBuffer1 := t.readObjectContent(context.Background(),
&gcs.ReadObjectRequest{
Name: TestObjectName,
Range: &gcs.ByteRange{
Start: uint64(0),
Limit: uint64(len(ContentInTestObject)),
},
})
// Validation of srcObject2 to ensure that it is not effected.
srcBuffer2 := t.readObjectContent(context.Background(),
&gcs.ReadObjectRequest{
Name: TestSubObjectName,
Range: &gcs.ByteRange{
Start: uint64(0),
Limit: uint64(len(ContentInTestSubObject)),
},
})
// Reading content of dstObject
dstBuffer := t.readObjectContent(context.Background(),
&gcs.ReadObjectRequest{
Name: dstObjectName,
Range: &gcs.ByteRange{
Start: uint64(0),
Limit: uint64(composedObj.Size),
},
})
// Comparing content of destination object
ExpectEq(srcBuffer1+srcBuffer2, dstBuffer)
AssertNe(nil, composedObj)
AssertEq(srcObj1.Size+srcObj2.Size, composedObj.Size)
}

func (t *BucketHandleTest) TestComposeObjectMethodWhenSrcObjectDoesNotExist() {
var notfound *gcs.NotFoundError
_, err := t.bucketHandle.StatObject(context.Background(),
&gcs.StatObjectRequest{
Name: missingObjectName,
})
// SrcObject does not exist
AssertTrue(errors.As(err, &notfound))

_, err = t.bucketHandle.ComposeObjects(context.Background(),
&gcs.ComposeObjectsRequest{
DstName: TestObjectName,
DstGenerationPrecondition: nil,
DstMetaGenerationPrecondition: nil,
Sources: []gcs.ComposeSource{
{
Name: missingObjectName,
},
},
ContentType: ContentType,
Metadata: map[string]string{
MetaDataKey: MetaDataValue,
},
ContentLanguage: ContentLanguage,
ContentEncoding: ContentEncoding,
CacheControl: CacheControl,
ContentDisposition: ContentDisposition,
CustomTime: CustomTime,
EventBasedHold: true,
StorageClass: StorageClass,
Acl: nil,
})

// For fakeobject it is giving googleapi 500 error, where as in real mounting we are getting "404 not found error"
AssertNe(nil, err)
}

func (t *BucketHandleTest) TestComposeObjectMethodWhenSourceIsNil() {
_, err := t.bucketHandle.ComposeObjects(context.Background(),
&gcs.ComposeObjectsRequest{
DstName: TestObjectName,
DstGenerationPrecondition: nil,
DstMetaGenerationPrecondition: nil,
Sources: nil,
ContentType: ContentType,
Metadata: map[string]string{
MetaDataKey: MetaDataValue,
},
ContentLanguage: ContentLanguage,
ContentEncoding: ContentEncoding,
CacheControl: CacheControl,
ContentDisposition: ContentDisposition,
CustomTime: CustomTime,
EventBasedHold: true,
StorageClass: StorageClass,
Acl: nil,
})

// error : Error in composing object: storage: at least one source object must be specified
AssertNe(nil, err)
}

func (t *BucketHandleTest) TestNameMethod() {
name := t.bucketHandle.Name()

Expand Down
3 changes: 2 additions & 1 deletion internal/storage/fake_storage_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ const TestObjectName string = "gcsfuse/default.txt"
const TestObjectSubRootFolderName string = "gcsfuse/SubFolder/"
const TestSubObjectName string = "gcsfuse/SubFolder/default.txt"
const ContentInTestObject string = "Hello GCSFuse!!!"
const ContentInTestSubObject string = "Hello GCSFuse From SubObject!!!"
const TestObjectGeneration int64 = 780
const MetaDataValue string = "metaData"
const MetaDataKey string = "key"
Expand Down Expand Up @@ -97,7 +98,7 @@ func getTestFakeStorageObject() []fakestorage.Object {
Name: TestSubObjectName,
Generation: TestObjectGeneration,
},
Content: []byte(ContentInTestObject),
Content: []byte(ContentInTestSubObject),
}
fakeObjects = append(fakeObjects, testSubObject)

Expand Down

0 comments on commit 74689e0

Please sign in to comment.