CORS - support AWS APIs (NC and containerized)

config.js

@@ -64,7 +64,7 @@ config.BUFFERS_MEM_LIMIT_MIN = 32 * 1024 * 1024; // just some workable minimum s
 config.BUFFERS_MEM_LIMIT_MAX = 4 * 1024 * 1024 * 1024;
 config.BUFFERS_MEM_LIMIT = Math.min(
     config.BUFFERS_MEM_LIMIT_MAX,
-    Math.max(Math.floor(config.CONTAINER_MEM_LIMIT / 4), config.BUFFERS_MEM_LIMIT_MIN, )
+    Math.max(Math.floor(config.CONTAINER_MEM_LIMIT / 4), config.BUFFERS_MEM_LIMIT_MIN)
 );
 
 ////////////////////////
@@ -152,9 +152,8 @@ config.ENDPOINT_HTTP_SERVER_REQUEST_TIMEOUT = 300 * 1000;
 config.ENDPOINT_HTTP_SERVER_KEEPALIVE_TIMEOUT = 5 * 1000;
 config.ENDPOINT_HTTP_MAX_REQUESTS_PER_SOCKET = 0; // 0 = no limit
 
-// For now we enable fixed CORS for all buckets
-// but this should become a setting per bucket which is configurable
-// with the s3 put-bucket-cors api.
+// For now we enable fixed CORS only for sts
+// for S3 per bucket is configurabl with the s3 put-bucket-cors api.
 // note that browsers do not really allow origin=* with credentials,
 // but we just allow both from our side for simplicity.
 config.S3_CORS_ENABLED = true;
@@ -502,6 +501,7 @@ config.LOG_COLOR_ENABLED = process.env.NOOBAA_LOG_COLOR ? process.env.NOOBAA_LOG
 
 // TEST Mode
 config.test_mode = false;
+config.allow_anonymous_access_in_test = false; // used for emulating ACL='public-read' for ceph-s3 tests
 
 // On Premise NVA params
 config.on_premise = {
@@ -753,11 +753,11 @@ config.NSFS_BUF_POOL_MEM_LIMIT_XS = Math.min(Math.floor(config.NSFS_MAX_MEM_SIZE
 config.NSFS_BUF_POOL_MEM_LIMIT_S = Math.min(Math.floor(config.NSFS_MAX_MEM_SIZE_S / config.NSFS_BUF_SIZE_S),
     config.NSFS_WANTED_BUFFERS_NUMBER) * config.NSFS_BUF_SIZE_S;
 // Semaphore size will give 90% of remainning memory to large buffer size, 10% to medium
-config.NSFS_BUF_POOL_MEM_LIMIT_M = range_utils.align_down((config.BUFFERS_MEM_LIMIT -
-        config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.1,
+config.NSFS_BUF_POOL_MEM_LIMIT_M = range_utils.align_down(
+    (config.BUFFERS_MEM_LIMIT - config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.1,
     config.NSFS_BUF_SIZE_M);
-config.NSFS_BUF_POOL_MEM_LIMIT_L = range_utils.align_down((config.BUFFERS_MEM_LIMIT -
-        config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.9,
+config.NSFS_BUF_POOL_MEM_LIMIT_L = range_utils.align_down(
+    (config.BUFFERS_MEM_LIMIT - config.NSFS_BUF_POOL_MEM_LIMIT_S - config.NSFS_BUF_POOL_MEM_LIMIT_XS) * 0.9,
     config.NSFS_BUF_SIZE_L);
 
 config.NSFS_BUF_WARMUP_SPARSE_FILE_READS = true;

src/api/bucket_api.js

@@ -343,7 +343,7 @@ module.exports = {
                     notifications: {
                         type: 'array',
                         items: {
-                             $ref: 'common_api#/definitions/bucket_notification'
+                            $ref: 'common_api#/definitions/bucket_notification'
                         }
                     }
                 }
@@ -890,6 +890,65 @@ module.exports = {
                 system: ['admin', 'user']
             }
         },
+
+        get_bucket_cors: {
+            method: 'GET',
+            params: {
+                type: 'object',
+                required: ['name'],
+                properties: {
+                    name: {
+                        $ref: 'common_api#/definitions/bucket_name'
+                    },
+                },
+            },
+            reply: {
+                type: 'object',
+                properties: {
+                    cors: {
+                        $ref: 'common_api#/definitions/bucket_cors_configuration'
+                    }
+                }
+            },
+            auth: {
+                system: ['admin', 'user']
+            }
+        },
+
+        put_bucket_cors: {
+            method: 'PUT',
+            params: {
+                type: 'object',
+                required: ['name', 'cors_rules'],
+                properties: {
+                    name: {
+                        $ref: 'common_api#/definitions/bucket_name'
+                    },
+                    cors_rules: {
+                        $ref: 'common_api#/definitions/bucket_cors_configuration'
+                    },
+                },
+            },
+            auth: {
+                system: ['admin', 'user']
+            }
+        },
+
+        delete_bucket_cors: {
+            method: 'DELETE',
+            params: {
+                type: 'object',
+                required: ['name'],
+                properties: {
+                    name: {
+                        $ref: 'common_api#/definitions/bucket_name'
+                    },
+                },
+            },
+            auth: {
+                system: ['admin', 'user']
+            }
+        },
     },
 
     definitions: {
@@ -1200,6 +1259,9 @@ module.exports = {
                     items: {
                         $ref: 'common_api#/definitions/bucket_notification'
                     }
+                },
+                cors_configuration_rules: {
+                    $ref: 'common_api#/definitions/bucket_cors_configuration',
                 }
             }
         },

src/api/common_api.js

@@ -271,6 +271,50 @@ module.exports = {
             }
         },
 
+        bucket_cors_configuration: {
+            type: 'array',
+            items: {
+                $ref: '#/definitions/bucket_cors_rule'
+            }
+        },
+
+        bucket_cors_rule: {
+            type: 'object',
+            required: ['allowed_methods', 'allowed_origins'],
+            properties: {
+                id: {
+                    type: 'string'
+                },
+                allowed_methods: {
+                    type: 'array',
+                    items: {
+                        type: 'string'
+                    }
+                },
+                allowed_origins: {
+                    type: 'array',
+                    items: {
+                        type: 'string'
+                    }
+                },
+                allowed_headers: {
+                    type: 'array',
+                    items: {
+                        type: 'string'
+                    }
+                },
+                expose_headers: {
+                    type: 'array',
+                    items: {
+                        type: 'string'
+                    }
+                },
+                max_age_seconds: {
+                    type: 'integer'
+                },
+            }
+        },
+
         bucket_policy_principal: {
             anyOf: [{
                 wrapper: SensitiveString

src/endpoint/s3/ops/s3_delete_bucket_cors.js

@@ -5,8 +5,9 @@
  * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketDELETEcors.html
  */
 async function delete_bucket_cors(req) {
-    await req.object_sdk.read_bucket({ name: req.params.bucket });
-    // TODO S3 delete_bucket_cors not implemented
+    await req.object_sdk.delete_bucket_cors({
+        name: req.params.bucket,
+    });
 }
 
 module.exports = {

src/endpoint/s3/ops/s3_get_bucket_cors.js

@@ -1,14 +1,25 @@
 /* Copyright (C) 2016 NooBaa */
 'use strict';
 
+const S3Error = require('../s3_errors').S3Error;
+
 /**
  * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETcors.html
  */
 async function get_bucket_cors(req) {
-    await req.object_sdk.read_bucket({ name: req.params.bucket });
-    return {
-        CORSConfiguration: ''
-    };
+    const reply = await req.object_sdk.get_bucket_cors({ name: req.params.bucket });
+    if (!reply.cors.length) throw new S3Error(S3Error.NoSuchCORSConfiguration);
+    const cors_rules = reply.cors.map(rule => {
+        const new_rule = [];
+        new_rule.push(...rule.allowed_methods.map(m => ({ AllowedMethod: m })));
+        new_rule.push(...rule.allowed_origins.map(o => ({ AllowedOrigin: o })));
+        if (rule.allowed_headers) new_rule.push(...rule.allowed_headers.map(h => ({ AllowedHeader: h })));
+        if (rule.expose_headers) new_rule.push(...rule.expose_headers.map(e => ({ ExposeHeader: e })));
+        if (rule.id) new_rule.push({ ID: rule.id });
+        if (rule.max_age_seconds) new_rule.push({ MaxAgeSeconds: rule.max_age_seconds });
+        return { CORSRule: new_rule };
+    });
+    return { CORSConfiguration: cors_rules.length ? cors_rules : '' };
 }
 
 module.exports = {

src/endpoint/s3/ops/s3_put_bucket.js

@@ -2,13 +2,26 @@
 'use strict';
 const config = require('../../../../config');
 
+
 /**
  * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUT.html
  */
 async function put_bucket(req, res) {
     const lock_enabled = config.WORM_ENABLED ? req.headers['x-amz-bucket-object-lock-enabled'] &&
         req.headers['x-amz-bucket-object-lock-enabled'].toUpperCase() === 'TRUE' : undefined;
     await req.object_sdk.create_bucket({ name: req.params.bucket, lock_enabled: lock_enabled });
+    if (config.allow_anonymous_access_in_test && req.headers['x-amz-acl'] === 'public-read') { // For now we will enable only for tests
+        const policy = {
+            Version: '2012-10-17',
+            Statement: [{
+                Effect: 'Allow',
+                Principal: { AWS: ["*"] },
+                Action: ['s3:GetObject', 's3:ListBucket'],
+                Resource: ['arn:aws:s3:::*']
+            }]
+        };
+        await req.object_sdk.put_bucket_policy({ name: req.params.bucket, policy });
+    }
     res.setHeader('Location', '/' + req.params.bucket);
 }
 

src/endpoint/s3/ops/s3_put_bucket_cors.js

@@ -1,15 +1,26 @@
 /* Copyright (C) 2016 NooBaa */
 'use strict';
 
-const S3Error = require('../s3_errors').S3Error;
+const _ = require('lodash');
 
 /**
  * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTcors.html
  */
 async function put_bucket_cors(req) {
-    await req.object_sdk.read_bucket({ name: req.params.bucket });
-    // TODO S3 put_bucket_cors not implemented
-    throw new S3Error(S3Error.NotImplemented);
+    const cors_rules = req.body.CORSConfiguration.CORSRule.map(rule =>
+        _.omitBy({
+            allowed_headers: rule.AllowedHeader,
+            allowed_methods: rule.AllowedMethod,
+            allowed_origins: rule.AllowedOrigin,
+            expose_headers: rule.ExposeHeader,
+            id: rule.ID,
+            max_age_seconds: rule.MaxAgeSeconds,
+        }, _.isUndefined)
+    );
+    await req.object_sdk.put_bucket_cors({
+        name: req.params.bucket,
+        cors_rules
+    });
 }
 
 module.exports = {

src/endpoint/s3/s3_errors.js

@@ -339,6 +339,11 @@ S3Error.NoSuchLifecycleConfiguration = Object.freeze({
     message: 'The lifecycle configuration does not exist.',
     http_code: 404,
 });
+S3Error.NoSuchCORSConfiguration = Object.freeze({
+    code: 'NoSuchCORSConfiguration',
+    message: 'The specified bucket does not have a CORS configuration.',
+    http_code: 404,
+});
 S3Error.NoSuchUpload = Object.freeze({
     code: 'NoSuchUpload',
     message: 'The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.',

src/endpoint/s3/s3_rest.js

@@ -83,14 +83,6 @@ async function handle_request(req, res) {
 
     http_utils.validate_server_ip_whitelist(req);
     http_utils.set_amz_headers(req, res);
-    http_utils.set_cors_headers_s3(req, res);
-
-    if (req.method === 'OPTIONS') {
-        dbg.log1('OPTIONS!');
-        res.statusCode = 200;
-        res.end();
-        return;
-    }
 
     const headers_options = {
         ErrorClass: S3Error,
@@ -115,6 +107,18 @@ async function handle_request(req, res) {
     }
 
     const op_name = parse_op_name(req);
+    const cors = req.params.bucket && await req.object_sdk.read_bucket_sdk_cors_info(req.params.bucket);
+
+    http_utils.set_cors_headers_s3(req, res, cors);
+
+    if (req.method === 'OPTIONS') {
+        dbg.log1('OPTIONS!');
+        const error_code = req.headers.origin && req.headers['access-control-request-method'] ? 403 : 400;
+        const res_headers = res.getHeaders(); // We will check if we found a matching rule - if no we will return error_code
+        res.statusCode = res_headers['access-control-allow-origin'] && res_headers['access-control-allow-methods'] ? 200 : error_code;
+        res.end();
+        return;
+    }
     const op = s3_ops[op_name];
     if (!op || !op.handler) {
         dbg.error('S3 NotImplemented', op_name, req.method, req.originalUrl);

src/sdk/bucketspace_fs.js

@@ -697,6 +697,47 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
         }
     }
 
+    ////////////////////
+    // BUCKET CORS //
+    ////////////////////
+
+    async put_bucket_cors(params) {
+        try {
+            const { name, cors_rules } = params;
+            dbg.log0('BucketSpaceFS.put_bucket_cors: Bucket name', name, ", cors configuration ", cors_rules);
+            const bucket = await this.config_fs.get_bucket_by_name(name);
+            bucket.cors_configuration_rules = cors_rules;
+            await this.config_fs.update_bucket_config_file(bucket);
+        } catch (error) {
+            throw translate_error_codes(error, entity_enum.BUCKET);
+        }
+    }
+
+    async delete_bucket_cors(params) {
+        try {
+            const { name } = params;
+            dbg.log0('BucketSpaceFS.delete_bucket_cors: Bucket name', name);
+            const bucket = await this.config_fs.get_bucket_by_name(name);
+            delete bucket.cors_configuration_rules;
+            await this.config_fs.update_bucket_config_file(bucket);
+        } catch (err) {
+            throw translate_error_codes(err, entity_enum.BUCKET);
+        }
+    }
+
+    async get_bucket_cors(params) {
+        try {
+            const { name } = params;
+            dbg.log0('BucketSpaceFS.get_bucket_cors: Bucket name', name);
+            const bucket = await this.config_fs.get_bucket_by_name(name);
+            return {
+                cors: bucket.cors_configuration_rules || [],
+            };
+        } catch (error) {
+            throw translate_error_codes(error, entity_enum.BUCKET);
+        }
+    }
+
     /////////////////////////
     // DEFAULT OBJECT LOCK //
     /////////////////////////