CORS

Signed-off-by: jackyalbo <[email protected]>
noobaa · Nov 18, 2024 · b99b160 · b99b160
1 parent e22503b
commit b99b160
Show file tree

Hide file tree

Showing 18 changed files with 336 additions and 70 deletions.
diff --git a/config.js b/config.js
@@ -64,7 +64,7 @@ config.BUFFERS_MEM_LIMIT_MIN = 32 * 1024 * 1024; // just some workable minimum s
 config.BUFFERS_MEM_LIMIT_MAX = 4 * 1024 * 1024 * 1024;
 config.BUFFERS_MEM_LIMIT = Math.min(
     config.BUFFERS_MEM_LIMIT_MAX,
-    Math.max(Math.floor(config.CONTAINER_MEM_LIMIT / 4), config.BUFFERS_MEM_LIMIT_MIN,)
+    Math.max(Math.floor(config.CONTAINER_MEM_LIMIT / 4), config.BUFFERS_MEM_LIMIT_MIN, )
 );
 
 ////////////////////////
@@ -503,6 +503,7 @@ config.LOG_COLOR_ENABLED = process.env.NOOBAA_LOG_COLOR ? process.env.NOOBAA_LOG
 
 // TEST Mode
 config.test_mode = false;
+config.allow_anonymous_access_in_test = false; // used for emulating ACL='public-read-write' for ceph-s3 tests
 
 // On Premise NVA params
 config.on_premise = {
@@ -751,11 +752,11 @@ config.NSFS_BUF_POOL_MEM_LIMIT_XS = Math.min(Math.floor(config.NSFS_MAX_MEM_SIZE
 config.NSFS_BUF_POOL_MEM_LIMIT_S = Math.min(Math.floor(config.NSFS_MAX_MEM_SIZE_S / config.NSFS_BUF_SIZE_S),
     config.NSFS_WANTED_BUFFERS_NUMBER) * config.NSFS_BUF_SIZE_S;
 // Semaphore size will give 90% of remainning memory to large buffer size, 10% to medium
-config.NSFS_BUF_POOL_MEM_LIMIT_M = range_utils.align_down((config.BUFFERS_MEM_LIMIT -
-    config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.1,
+config.NSFS_BUF_POOL_MEM_LIMIT_M = range_utils.align_down(
+    (config.BUFFERS_MEM_LIMIT - config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.1,
     config.NSFS_BUF_SIZE_M);
-config.NSFS_BUF_POOL_MEM_LIMIT_L = range_utils.align_down((config.BUFFERS_MEM_LIMIT -
-    config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.9,
+config.NSFS_BUF_POOL_MEM_LIMIT_L = range_utils.align_down(
+    (config.BUFFERS_MEM_LIMIT - config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.9,
     config.NSFS_BUF_SIZE_L);
 
 config.NSFS_BUF_WARMUP_SPARSE_FILE_READS = true;

diff --git a/src/api/bucket_api.js b/src/api/bucket_api.js
@@ -343,7 +343,7 @@ module.exports = {
                     notifications: {
                         type: 'array',
                         items: {
-                             $ref: 'common_api#/definitions/bucket_notification'
+                            $ref: 'common_api#/definitions/bucket_notification'
                         }
                     }
                 }
@@ -890,6 +890,60 @@ module.exports = {
                 system: ['admin', 'user']
             }
         },
+
+        get_bucket_cors: {
+            method: 'GET',
+            params: {
+                type: 'object',
+                required: ['name'],
+                properties: {
+                    name: {
+                        $ref: 'common_api#/definitions/bucket_name'
+                    },
+                },
+            },
+            reply: {
+                $ref: 'common_api#/definitions/bucket_cors_configuration',
+            },
+            auth: {
+                system: ['admin', 'user']
+            }
+        },
+
+        put_bucket_cors: {
+            method: 'PUT',
+            params: {
+                type: 'object',
+                required: ['name', 'cors_rules'],
+                properties: {
+                    name: {
+                        $ref: 'common_api#/definitions/bucket_name'
+                    },
+                    cors_rules: {
+                        $ref: 'common_api#/definitions/bucket_cors_configuration'
+                    },
+                },
+            },
+            auth: {
+                system: ['admin', 'user']
+            }
+        },
+
+        delete_bucket_cors: {
+            method: 'DELETE',
+            params: {
+                type: 'object',
+                required: ['name'],
+                properties: {
+                    name: {
+                        $ref: 'common_api#/definitions/bucket_name'
+                    },
+                },
+            },
+            auth: {
+                system: ['admin', 'user']
+            }
+        },
     },
 
     definitions: {
@@ -1200,6 +1254,9 @@ module.exports = {
                     items: {
                         $ref: 'common_api#/definitions/bucket_notification'
                     }
+                },
+                cors_configuration_rules: {
+                    $ref: 'common_api#/definitions/bucket_cors_configuration',
                 }
             }
         },

diff --git a/src/api/common_api.js b/src/api/common_api.js
@@ -271,6 +271,50 @@ module.exports = {
             }
         },
 
+        bucket_cors_configuration: {
+            type: 'array',
+            items: {
+                $ref: '#/definitions/bucket_cors_rule'
+            }
+        },
+
+        bucket_cors_rule: {
+            type: 'object',
+            required: ['allowed_methods', 'allowed_origins'],
+            properties: {
+                id: {
+                    type: 'string'
+                },
+                allowed_methods: {
+                    type: 'array',
+                    items: {
+                        type: 'string'
+                    }
+                },
+                allowed_origins: {
+                    type: 'array',
+                    items: {
+                        type: 'string'
+                    }
+                },
+                allowed_headers: {
+                    type: 'array',
+                    items: {
+                        type: 'string'
+                    }
+                },
+                expose_headers: {
+                    type: 'array',
+                    items: {
+                        type: 'string'
+                    }
+                },
+                max_age_seconds: {
+                    type: 'integer'
+                },
+            }
+        },
+
         bucket_policy_principal: {
             anyOf: [{
                 wrapper: SensitiveString

diff --git a/src/endpoint/s3/ops/s3_delete_bucket_cors.js b/src/endpoint/s3/ops/s3_delete_bucket_cors.js
@@ -5,8 +5,9 @@
  * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketDELETEcors.html
  */
 async function delete_bucket_cors(req) {
-    await req.object_sdk.read_bucket({ name: req.params.bucket });
-    // TODO S3 delete_bucket_cors not implemented
+    await req.object_sdk.delete_bucket_cors({
+        name: req.params.bucket,
+    });
 }
 
 module.exports = {

diff --git a/src/endpoint/s3/ops/s3_get_bucket_cors.js b/src/endpoint/s3/ops/s3_get_bucket_cors.js
@@ -1,14 +1,26 @@
 /* Copyright (C) 2016 NooBaa */
 'use strict';
 
+const _ = require('lodash');
+const S3Error = require('../s3_errors').S3Error;
+
 /**
  * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETcors.html
  */
 async function get_bucket_cors(req) {
-    await req.object_sdk.read_bucket({ name: req.params.bucket });
-    return {
-        CORSConfiguration: ''
-    };
+    const reply = await req.object_sdk.get_bucket_cors({ name: req.params.bucket });
+    if (!reply.length) throw new S3Error(S3Error.NoSuchCORSConfiguration);
+    const cors_rules = reply.map(rule => {
+        const new_rule = [];
+        new_rule.push(...rule.allowed_methods.map(m => ({ AllowedMethod: m })));
+        new_rule.push(...rule.allowed_origins.map(o => ({ AllowedOrigin: o })));
+        if (rule.allowed_headers) new_rule.push(...rule.allowed_headers.map(h => ({ AllowedHeader: h })));
+        if (rule.expose_headers) new_rule.push(...rule.expose_headers.map(e => ({ ExposeHeader: e })));
+        if (rule.id) new_rule.push({ ID: rule.id });
+        if (rule.max_age_seconds) new_rule.push({ MaxAgeSeconds: rule.max_age_seconds });
+        return { CORSRule: new_rule };
+    });
+    return { CORSConfiguration: cors_rules.length ? cors_rules : '' };
 }
 
 module.exports = {

diff --git a/src/endpoint/s3/ops/s3_put_bucket_cors.js b/src/endpoint/s3/ops/s3_put_bucket_cors.js
@@ -1,15 +1,26 @@
 /* Copyright (C) 2016 NooBaa */
 'use strict';
 
-const S3Error = require('../s3_errors').S3Error;
+const _ = require('lodash');
 
 /**
  * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTcors.html
  */
 async function put_bucket_cors(req) {
-    await req.object_sdk.read_bucket({ name: req.params.bucket });
-    // TODO S3 put_bucket_cors not implemented
-    throw new S3Error(S3Error.NotImplemented);
+    const cors_rules = req.body.CORSConfiguration.CORSRule.map(rule =>
+        _.omitBy({
+            allowed_headers: rule.AllowedHeader,
+            allowed_methods: rule.AllowedMethod,
+            allowed_origins: rule.AllowedOrigin,
+            expose_headers: rule.ExposeHeader,
+            id: rule.ID,
+            max_age_seconds: rule.MaxAgeSeconds,
+        }, _.isUndefined)
+    );
+    await req.object_sdk.put_bucket_cors({
+        name: req.params.bucket,
+        cors_rules
+    });
 }
 
 module.exports = {

diff --git a/src/endpoint/s3/s3_errors.js b/src/endpoint/s3/s3_errors.js
@@ -339,6 +339,11 @@ S3Error.NoSuchLifecycleConfiguration = Object.freeze({
     message: 'The lifecycle configuration does not exist.',
     http_code: 404,
 });
+S3Error.NoSuchCORSConfiguration = Object.freeze({
+    code: 'NoSuchCORSConfiguration',
+    message: 'The specified bucket does not have a CORS configuration.',
+    http_code: 404,
+});
 S3Error.NoSuchUpload = Object.freeze({
     code: 'NoSuchUpload',
     message: 'The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.',

diff --git a/src/endpoint/s3/s3_rest.js b/src/endpoint/s3/s3_rest.js
@@ -83,14 +83,6 @@ async function handle_request(req, res) {
 
     http_utils.validate_server_ip_whitelist(req);
     http_utils.set_amz_headers(req, res);
-    http_utils.set_cors_headers_s3(req, res);
-
-    if (req.method === 'OPTIONS') {
-        dbg.log1('OPTIONS!');
-        res.statusCode = 200;
-        res.end();
-        return;
-    }
 
     const headers_options = {
         ErrorClass: S3Error,
@@ -115,6 +107,15 @@ async function handle_request(req, res) {
     }
 
     const op_name = parse_op_name(req);
+    const cors = req.params.bucket && await req.object_sdk.read_bucket_sdk_cors_info(req.params.bucket);
+
+    http_utils.set_cors_headers_s3(req, res, cors);
+
+    if (req.method === 'OPTIONS') {
+        dbg.log1('OPTIONS!');
+        res.end();
+        return;
+    }
     const op = s3_ops[op_name];
     if (!op || !op.handler) {
         dbg.error('S3 NotImplemented', op_name, req.method, req.originalUrl);
@@ -224,6 +225,7 @@ async function authorize_request_policy(req) {
 
     const is_anon = !(auth_token && auth_token.access_key);
     if (is_anon) {
+        if (config.allow_anonymous_access_in_test) return;
         await authorize_anonymous_access(s3_policy, method, arn_path, req);
         return;
     }

diff --git a/src/sdk/bucketspace_nb.js b/src/sdk/bucketspace_nb.js
@@ -33,7 +33,7 @@ class BucketSpaceNB {
     ////////////
 
     async list_buckets(params, object_sdk) {
-        const { buckets, continuation_token} = (await this.rpc_client.bucket.list_buckets(params));
+        const { buckets, continuation_token } = (await this.rpc_client.bucket.list_buckets(params));
 
         const has_access_buckets = (await P.all(_.map(
             buckets,
@@ -44,7 +44,7 @@ class BucketSpaceNB {
                     object_sdk.has_non_nsfs_bucket_access(object_sdk.requesting_account, ns);
                 return has_access_to_bucket && bucket;
             }))).filter(bucket => bucket);
-        return { buckets: has_access_buckets, continuation_token};
+        return { buckets: has_access_buckets, continuation_token };
     }
 
     async read_bucket(params) {
@@ -247,6 +247,29 @@ class BucketSpaceNB {
         });
     }
 
+    ////////////////////
+    // BUCKET CORS //
+    ////////////////////
+
+    async put_bucket_cors(params) {
+        return this.rpc_client.bucket.put_bucket_cors({
+            name: params.name,
+            cors_rules: params.cors_rules
+        });
+    }
+
+    async delete_bucket_cors(params) {
+        return this.rpc_client.bucket.delete_bucket_cors({
+            name: params.name
+        });
+    }
+
+    async get_bucket_cors(params) {
+        return this.rpc_client.bucket.get_bucket_cors({
+            name: params.name
+        });
+    }
+
     /////////////////////////
     // DEFAULT OBJECT LOCK //
     /////////////////////////

diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts
@@ -859,6 +859,10 @@ interface BucketSpace {
     put_bucket_notification(params: object): Promise<any>;
     get_bucket_notification(params: object): Promise<any>;
 
+    put_bucket_cors(params: object): Promise<any>;
+    delete_bucket_cors(params: object): Promise<any>;
+    get_bucket_cors(params: object): Promise<any>;
+
     get_object_lock_configuration(params: object, object_sdk: ObjectSDK): Promise<any>;
     put_object_lock_configuration(params: object, object_sdk: ObjectSDK): Promise<any>;