From d44419ed100e4435f416eaf82ecf8e66435505e5 Mon Sep 17 00:00:00 2001 From: Prakash Senthil Vel <23444145+prakashsvmx@users.noreply.github.com> Date: Wed, 2 Jun 2021 05:55:20 +0530 Subject: [PATCH] Add Bucket replication APIs (#934) --- README.md | 3 + docs/API.md | 99 ++++++++++++++++++++ examples/get-bucket-replication.js | 33 +++++++ examples/remove-bucket-replication.js | 33 +++++++ examples/set-bucket-replication.js | 54 +++++++++++ src/main/minio.js | 114 ++++++++++++++++++++---- src/main/transformers.js | 4 + src/main/xml-parsers.js | 11 ++- src/test/functional/functional-tests.js | 7 ++ src/test/unit/test.js | 104 +++++++++++++++++++++ 10 files changed, 443 insertions(+), 19 deletions(-) create mode 100644 examples/get-bucket-replication.js create mode 100644 examples/remove-bucket-replication.js create mode 100644 examples/set-bucket-replication.js diff --git a/README.md b/README.md index b29ea635..1420e2ec 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,9 @@ The full API Reference is available here. * [remove-bucket-lifecycle.js](https://github.com/minio/minio-js/blob/master/examples/remove-bucket-lifecycle.js) * [get-object-lock-config.js](https://github.com/minio/minio-js/blob/master/examples/get-object-lock-config.js) * [set-object-lock-config.js](https://github.com/minio/minio-js/blob/master/examples/set-object-lock-config.js) +* [set-bucket-replication.js](https://github.com/minio/minio-js/blob/master/examples/set-bucket-replication.js) +* [get-bucket-replication.js](https://github.com/minio/minio-js/blob/master/examples/get-bucket-replication.js) +* [remove-bucket-replication.js](https://github.com/minio/minio-js/blob/master/examples/remove-bucket-replication.js) #### Full Examples : File Object Operations * [fput-object.js](https://github.com/minio/minio-js/blob/master/examples/fput-object.js) diff --git a/docs/API.md b/docs/API.md index fd51f13f..c3505159 100644 --- a/docs/API.md +++ b/docs/API.md @@ -49,6 +49,10 @@ var s3Client = new Minio.Client({ |[`setBucketEncryption`](#setBucketEncryption) | | |[`getBucketEncryption`](#getBucketEncryption) | | |[`removeBucketEncryption`](#removeBucketEncryption) | | +| [`removeBucketLifecycle`](#removeBucketLifecycle) | | +| [`setBucketReplication`](#setBucketReplication)| | +| [`getBucketReplication`](#getBucketReplication)| | +| [`removeBucketReplication`](#removeBucketReplication)| | @@ -473,6 +477,101 @@ minioClient.setBucketVersioning('bucketname',versioningConfig, function (err){ }) ``` + + +### setBucketReplication(bucketName, replicationConfig, callback) + +Set replication config on a Bucket + +__Parameters__ + + +| Param | Type | Description | +| ---| ---|---| +| `bucketname` | _string_ | Name of the bucket. | +| `replicationConfig` | _object_ | replicationConfig Configuration as a JSON Object | +|`callback(err)` | _function_ | Callback is called with `err` in case of error.| + +__Example__ +```js +const arnFromMcCli = "arn:minio:replication::1277fcbe7df0bab76ab0c64cf7c45a0d27e01917ee5f11e913f3478417833660:destination" + +var replicationConfig = { + role:arnFromMcCli, + rules:[{ + "DeleteMarkerReplication": { + "Status": "Disabled" + }, + "DeleteReplication": { + "Status": [ + ] + }, + "Destination": { + "Bucket": "arn:aws:s3:::destination" + }, + "Priority": "1", + "Status": "Enabled" + }] +} + +minioClient.setBucketReplication('bucketname',replicationConfig, function (err){ + if (err) { + return console.log(err) + } + console.log("Success") +}) + +``` + + +### getBucketReplication(bucketName, callback) + +Get replication config of a Bucket + +__Parameters__ + + +| Param | Type | Description | +| ---| ---|---| +| `bucketname` | _string_ | Name of the bucket. | +|`callback(err, replicationConfig)` | _function_ | Callback is called with `err` in case of error. else returns the info in`replicationConfig` ,which contains`{role: __string__, rules:__Array__ }`. | + +__Example__ +```js +minioClient.getBucketReplication('bucketname', function (err,replicationConfig){ + if (err) { + return console.log(err) + } + console.log(replicationConfig) +}) + +``` + + + +### removeBucketReplication(bucketName, callback) + +Remove replication config of a Bucket + +__Parameters__ + + +| Param | Type | Description | +| ---| ---|---| +| `bucketname` | _string_ | Name of the bucket. | +|`callback(err)` | _function_ | Callback is called with `err` in case of error. | + +__Example__ +```js +minioClient.removeBucketReplication('bucketname', function (err,replicationConfig){ + if (err) { + return console.log(err) + } + console.log("Success") +}) + +``` + ### setBucketTagging(bucketName, tags, callback) diff --git a/examples/get-bucket-replication.js b/examples/get-bucket-replication.js new file mode 100644 index 00000000..3e37a595 --- /dev/null +++ b/examples/get-bucket-replication.js @@ -0,0 +1,33 @@ +/* + * MinIO Javascript Library for Amazon S3 Compatible Cloud Storage, (C) 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are +// dummy values, please replace them with original values. + +var Minio = require('minio') + +var s3Client = new Minio.Client({ + endPoint: 's3.amazonaws.com', + accessKey: 'YOUR-ACCESSKEYID', + secretKey: 'YOUR-SECRETACCESSKEY' +}) + +s3Client.getBucketReplication('bucketname', function (err, replicationConfig){ + if (err) { + return console.log(err) + } + console.log(replicationConfig) +}) diff --git a/examples/remove-bucket-replication.js b/examples/remove-bucket-replication.js new file mode 100644 index 00000000..c22baf90 --- /dev/null +++ b/examples/remove-bucket-replication.js @@ -0,0 +1,33 @@ +/* + * MinIO Javascript Library for Amazon S3 Compatible Cloud Storage, (C) 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are +// dummy values, please replace them with original values. + +var Minio = require('minio') + +var s3Client = new Minio.Client({ + endPoint: 's3.amazonaws.com', + accessKey: 'YOUR-ACCESSKEYID', + secretKey: 'YOUR-SECRETACCESSKEY' +}) + +s3Client.removeBucketReplication('bucketname', function (err){ + if (err) { + return console.log(err) + } + console.log("Success") +}) diff --git a/examples/set-bucket-replication.js b/examples/set-bucket-replication.js new file mode 100644 index 00000000..5268d7a9 --- /dev/null +++ b/examples/set-bucket-replication.js @@ -0,0 +1,54 @@ +/* + * MinIO Javascript Library for Amazon S3 Compatible Cloud Storage, (C) 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are +// dummy values, please replace them with original values. + +var Minio = require('minio') + +var s3Client = new Minio.Client({ + endPoint: 's3.amazonaws.com', + accessKey: 'YOUR-ACCESSKEYID', + secretKey: 'YOUR-SECRETACCESSKEY' +}) + + +const arnFromMcCli = "arn:minio:replication::1277fcbe7df0bab76ab0c64cf7c45a0d27e01917ee5f11e913f3478417833660:destination" + +var replicationConfig = { + role:arnFromMcCli, + rules:[{ + "DeleteMarkerReplication": { + "Status": "Disabled" + }, + "DeleteReplication": { + "Status": [ + ] + }, + "Destination": { + "Bucket": "arn:aws:s3:::destination" + }, + "Priority": "1", + "Status": "Enabled" + }] +} + +s3Client.setBucketReplication('bucketname',replicationConfig, function (err){ + if (err) { + return console.log(err) + } + console.log("Success") +}) diff --git a/src/main/minio.js b/src/main/minio.js index 50c2192e..caea391e 100644 --- a/src/main/minio.js +++ b/src/main/minio.js @@ -2860,6 +2860,81 @@ export class Client { } + setBucketReplication(bucketName, replicationConfig={}, cb) { + if (!isValidBucketName(bucketName)) { + throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName) + } + if (!isObject(replicationConfig)) { + throw new errors.InvalidArgumentError('replicationConfig should be of type "object"') + } else { + if (_.isEmpty(replicationConfig.role)) { + throw new errors.InvalidArgumentError('Role cannot be empty') + }else if (replicationConfig.role && !isString(replicationConfig.role)) { + throw new errors.InvalidArgumentError('Invalid value for role', replicationConfig.role) + } + if (_.isEmpty(replicationConfig.rules)) { + throw new errors.InvalidArgumentError('Minimum one replication rule must be specified') + } + } + if (!isFunction(cb)) { + throw new TypeError('callback should be of type "function"') + } + + const method = 'PUT' + let query = "replication" + const headers = {} + + const replicationParamsConfig = { + ReplicationConfiguration: { + Role: replicationConfig.role, + Rule: replicationConfig.rules + } + } + + const builder = new xml2js.Builder({ renderOpts:{'pretty':false},headless: true }) + + let payload = builder.buildObject(replicationParamsConfig) + + const md5digest = Crypto.createHash('md5').update(payload).digest() + headers['Content-MD5'] = md5digest.toString('base64') + + this.makeRequest({method, bucketName, query, headers}, payload, 200, '', false, cb) + } + + getBucketReplication(bucketName, cb) { + if (!isValidBucketName(bucketName)) { + throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName) + } + if (!isFunction(cb)) { + throw new errors.InvalidArgumentError('callback should be of type "function"') + } + const method = 'GET' + const query = "replication" + + this.makeRequest({method, bucketName, query}, '', 200, '', true, (e, response) => { + if (e) return cb(e) + + let replicationConfig = Buffer.from('') + pipesetup(response, transformers.replicationConfigTransformer()) + .on('data', data => { + replicationConfig = data + }) + .on('error', cb) + .on('end', () => { + cb(null, replicationConfig) + }) + }) + } + + removeBucketReplication(bucketName, cb){ + if (!isValidBucketName(bucketName)) { + throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName) + } + const method = 'DELETE' + const query="replication" + this.makeRequest({method, bucketName, query}, '', 200, '', false, cb) + } + get extensions() { if(!this.clientExtensions) { @@ -2895,24 +2970,27 @@ Client.prototype.removeAllBucketNotification = promisify(Client.prototype.remove Client.prototype.getBucketPolicy = promisify(Client.prototype.getBucketPolicy) Client.prototype.setBucketPolicy = promisify(Client.prototype.setBucketPolicy) Client.prototype.removeIncompleteUpload = promisify(Client.prototype.removeIncompleteUpload) -Client.prototype.getBucketVersioning = promisify((Client.prototype.getBucketVersioning)) -Client.prototype.setBucketVersioning=promisify((Client.prototype.setBucketVersioning)) -Client.prototype.setBucketTagging=promisify((Client.prototype.setBucketTagging)) -Client.prototype.removeBucketTagging=promisify((Client.prototype.removeBucketTagging)) -Client.prototype.getBucketTagging=promisify((Client.prototype.getBucketTagging)) -Client.prototype.setObjectTagging=promisify((Client.prototype.setObjectTagging)) -Client.prototype.removeObjectTagging=promisify((Client.prototype.removeObjectTagging)) -Client.prototype.getObjectTagging=promisify((Client.prototype.getObjectTagging)) -Client.prototype.setBucketLifecycle=promisify((Client.prototype.setBucketLifecycle)) -Client.prototype.getBucketLifecycle=promisify((Client.prototype.getBucketLifecycle)) -Client.prototype.removeBucketLifecycle=promisify((Client.prototype.removeBucketLifecycle)) -Client.prototype.setObjectLockConfig=promisify((Client.prototype.setObjectLockConfig)) -Client.prototype.getObjectLockConfig=promisify((Client.prototype.getObjectLockConfig)) -Client.prototype.putObjectRetention =promisify((Client.prototype.putObjectRetention)) -Client.prototype.getObjectRetention =promisify((Client.prototype.getObjectRetention)) -Client.prototype.setBucketEncryption = promisify((Client.prototype.setBucketEncryption)) -Client.prototype.getBucketEncryption = promisify((Client.prototype.getBucketEncryption)) -Client.prototype.removeBucketEncryption = promisify((Client.prototype.removeBucketEncryption)) +Client.prototype.getBucketVersioning = promisify(Client.prototype.getBucketVersioning) +Client.prototype.setBucketVersioning=promisify(Client.prototype.setBucketVersioning) +Client.prototype.setBucketTagging=promisify(Client.prototype.setBucketTagging) +Client.prototype.removeBucketTagging=promisify(Client.prototype.removeBucketTagging) +Client.prototype.getBucketTagging=promisify(Client.prototype.getBucketTagging) +Client.prototype.setObjectTagging=promisify(Client.prototype.setObjectTagging) +Client.prototype.removeObjectTagging=promisify(Client.prototype.removeObjectTagging) +Client.prototype.getObjectTagging=promisify(Client.prototype.getObjectTagging) +Client.prototype.setBucketLifecycle=promisify(Client.prototype.setBucketLifecycle) +Client.prototype.getBucketLifecycle=promisify(Client.prototype.getBucketLifecycle) +Client.prototype.removeBucketLifecycle=promisify(Client.prototype.removeBucketLifecycle) +Client.prototype.setObjectLockConfig=promisify(Client.prototype.setObjectLockConfig) +Client.prototype.getObjectLockConfig=promisify(Client.prototype.getObjectLockConfig) +Client.prototype.putObjectRetention =promisify(Client.prototype.putObjectRetention) +Client.prototype.getObjectRetention =promisify(Client.prototype.getObjectRetention) +Client.prototype.setBucketEncryption = promisify(Client.prototype.setBucketEncryption) +Client.prototype.getBucketEncryption = promisify(Client.prototype.getBucketEncryption) +Client.prototype.removeBucketEncryption = promisify(Client.prototype.removeBucketEncryption) +Client.prototype.setBucketReplication =promisify(Client.prototype.setBucketReplication) +Client.prototype.getBucketReplication =promisify(Client.prototype.getBucketReplication) +Client.prototype.removeBucketReplication=promisify(Client.prototype.removeBucketReplication) export class CopyConditions { constructor() { diff --git a/src/main/transformers.js b/src/main/transformers.js index 042da76e..a84f87d0 100644 --- a/src/main/transformers.js +++ b/src/main/transformers.js @@ -236,4 +236,8 @@ export function objectRetentionTransformer(){ } export function bucketEncryptionTransformer(){ return getConcater(xmlParsers.parseBucketEncryptionConfig) +} + +export function replicationConfigTransformer(){ + return getConcater(xmlParsers.parseReplicationConfig) } \ No newline at end of file diff --git a/src/main/xml-parsers.js b/src/main/xml-parsers.js index 720f3134..8734b163 100644 --- a/src/main/xml-parsers.js +++ b/src/main/xml-parsers.js @@ -523,8 +523,17 @@ export function parseObjectRetentionConfig(xml){ } } - export function parseBucketEncryptionConfig(xml){ let encConfig = parseXml(xml) return encConfig +} +export function parseReplicationConfig(xml){ + const xmlObj = parseXml(xml) + const replicationConfig = { + ReplicationConfiguration: { + role: xmlObj.ReplicationConfiguration.Role, + rules: toArray(xmlObj.ReplicationConfiguration.Rule) + } + } + return replicationConfig } \ No newline at end of file diff --git a/src/test/functional/functional-tests.js b/src/test/functional/functional-tests.js index 2c2805df..e476bbd4 100644 --- a/src/test/functional/functional-tests.js +++ b/src/test/functional/functional-tests.js @@ -2367,4 +2367,11 @@ describe('functional tests', function() { }) + describe('Bucket Replication API Tests', ()=> { + //TODO - As of now, there is no api to get arn programmatically to setup replication through APIs and verify. + //Please refer to minio server documentation and mc cli. + //https://docs.min.io/docs/minio-bucket-replication-guide.html + //https://docs.min.io/minio/baremetal/replication/replication-overview.html#minio-bucket-replication-clientside + }) + }) diff --git a/src/test/unit/test.js b/src/test/unit/test.js index 1ac1e927..92f2f441 100644 --- a/src/test/unit/test.js +++ b/src/test/unit/test.js @@ -1238,5 +1238,109 @@ describe('Client', function() { }) }) + describe('Bucket Replication APIs', ()=> { + describe('setBucketReplication(bucketName, replicationConfig, callback)', () => { + it('should fail on null bucket', (done) => { + try { + client.setBucketReplication(null, {}, function () { + }) + } catch (e) { + done() + } + }) + it('should fail on empty bucket', (done) => { + try { + client.setBucketReplication('', {}, function () { + }) + } catch (e) { + done() + } + }) + + it('should fail on empty replicationConfig', (done) => { + try { + client.setBucketReplication('my-bucket', {}, function () { + }) + } catch (e) { + done() + } + }) + + it('should fail on empty replicationConfig role', (done) => { + try { + client.setBucketReplication('my-bucket', {role:''}, function () { + }) + } catch (e) { + done() + } + }) + + it('should fail on invalid value for replicationConfig role', (done) => { + try { + client.setBucketReplication('my-bucket', {role:12}, function () { + }) + } catch (e) { + done() + } + }) + + it('should fail on empty value for replicationConfig rules', (done) => { + try { + client.setBucketReplication('my-bucket', {role:"arn:",rules:[]}, function () { + }) + } catch (e) { + done() + } + }) + it('should fail on null value for replicationConfig rules', (done) => { + try { + client.setBucketReplication('my-bucket', {role:"arn:",rules:null}, function () { + }) + } catch (e) { + done() + } + }) + + }) + + describe('getBucketReplication(bucketName, callback)', () => { + it('should fail on null bucket', (done) => { + try { + client.getBucketReplication(null, {}, function () { + }) + } catch (e) { + done() + } + }) + it('should fail on empty bucket', (done) => { + try { + client.getBucketReplication('', {},function () { + }) + } catch (e) { + done() + } + }) + }) + + describe('removeBucketReplication(bucketName, callback)', () => { + it('should fail on null bucket', (done) => { + try { + client.removeBucketReplication(null, {}, function () { + }) + } catch (e) { + done() + } + }) + it('should fail on empty bucket', (done) => { + try { + client.removeBucketReplication('', {},function () { + }) + } catch (e) { + done() + } + }) + }) + }) + })