Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add S3 SSE KMS key and bucket key encryption #28601

Closed
wants to merge 1 commit into from
Closed

Add S3 SSE KMS key and bucket key encryption #28601

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 131 additions & 21 deletions lib/private/Files/ObjectStore/S3ConnectionTrait.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
use Aws\S3\S3Client;
use GuzzleHttp\Promise;
use GuzzleHttp\Promise\RejectedPromise;
use OCP\ILogger;
use Psr\Log\LoggerInterface;

trait S3ConnectionTrait {
/** @var array */
Expand All @@ -63,11 +63,17 @@ trait S3ConnectionTrait {
/** @var int */
protected $uploadPartSize;

/** @var string */
protected $sseKmsKeyId;

/** @var bool */
protected $sseUseBucketKey;

protected $test;

protected function parseParams($params) {
if (empty($params['bucket'])) {
throw new \Exception("Bucket has to be configured.");
throw new \Exception('Bucket has to be configured.');
}

$this->id = 'amazon::' . $params['bucket'];
Expand All @@ -82,7 +88,11 @@ protected function parseParams($params) {
if (!isset($params['port']) || $params['port'] === '') {
$params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443;
}
$params['verify_bucket_exists'] = empty($params['verify_bucket_exists']) ? true : $params['verify_bucket_exists'];
$params['autocreate'] = $params['autocreate'] ?? false;

$this->sseKmsKeyId = $params['ssekmskeyid'] ?? null;
$this->sseUseBucketKey = $params['sseusebucketkey'] ?? false;

$this->params = $params;
}

Expand All @@ -95,7 +105,118 @@ public function getProxy() {
}

/**
* Returns the connection
* Add the SSE KMS parameterdepending on the
* KMS encryption strategy (bucket, individual or
* no encryption) for object creations.
*
* @return array with encryption parameters
*/
public function getSseKmsPutParameters(): array {
if ($this->sseUseBucketKey) {
return [
'ServerSideEncryption' => 'aws:kms',
'BucketKeyEnabled' => true,
];
}

if (!empty($this->sseKmsKeyId)) {
return [
'ServerSideEncryption' => 'aws:kms',
'BucketKeyEnabled' => false,
'SSEKMSKeyId' => $this->sseKmsKeyId,
];
}

return [];
}

/**
* Add the SSE KMS parameter depending on the
* KMS encryption strategy (bucket, individual or
* no encryption) for object read.
*
* @return array with encryption parameters
*/
public function getSseKmsGetParameters(): array {
if ($this->sseUseBucketKey || !empty($this->sseKmsKeyId)) {
return [
'ServerSideEncryption' => 'aws:kms',
];
}
return [];
}

/**
* Create the required bucket
*
* @throws \Exception if bucket creation fails
*/
protected function createNewBucket() {
$logger = \OC::$server->get(LoggerInterface::class);
artonge marked this conversation as resolved.
Show resolved Hide resolved
try {
$logger->info('Bucket "'.$this->bucket.'" does not exist - creating it.', ['app' => 'objectstore']);
if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
throw new \Exception('The bucket will not be created because the name is not dns compatible, please correct it: '.$this->bucket);
}
$this->connection->createBucket(['Bucket' => $this->bucket]);
$this->testTimeout();
} catch (S3Exception $e) {
$logger->error('Invalid remote storage.', [
'exception' => $e,
'app' => 'objectstore',
]);
throw new \Exception('Creation of bucket "'.$this->bucket.'" failed. '.$e->getMessage());
}
}

/**
* Check bucket key consistency or put bucket key if missing
* This operation only works for bucket owner or with
* s3:GetEncryptionConfiguration/s3:PutEncryptionConfiguration permission
*
* We recommend to use autocreate only on initial setup and
* use an S3:user only with object operation permission and no bucket operation permissions
* later with autocreate=false
*
* @throws \Exception if bucket key config is inconsistent or if putting the key fails
*/
protected function checkOrPutBucketKey() {
$logger = \OC::$server->get(LoggerInterface::class);

try {
$encryptState = $this->connection->getBucketEncryption([
'Bucket' => $this->bucket,
]);
} catch (S3Exception $e) {
try {
$logger->info('Bucket key for "'.$this->bucket.'" is not set - adding it.', ['app' => 'objectstore']);
$this->connection->putBucketEncryption([
'Bucket' => $this->bucket,
'ServerSideEncryptionConfiguration' => [
'Rules' => [
[
'ApplyServerSideEncryptionByDefault' => [
'SSEAlgorithm' => 'aws:kms',
'KMSMasterKeyID' => $this->sseKmsKeyId,
],
],
],
],
]);
$this->testTimeout();
} catch (S3Exception $e) {
$logger->error('Bucket key problem.', [
'exception' => $e,
'app' => 'objectstore',
]);
throw new \Exception('Putting configured bucket key to "' . $this->bucket . '" failed. ' . $e->getMessage());
}
}
}


/**
* Returns the connection.
*
* @return S3Client connected client
* @throws \Exception if connection could not be made
Expand Down Expand Up @@ -145,23 +266,12 @@ public function getConnection() {
['app' => 'objectstore']);
}

if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) {
$logger = \OC::$server->getLogger();
try {
$logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']);
if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
throw new \Exception("The bucket will not be created because the name is not dns compatible, please correct it: " . $this->bucket);
}
$this->connection->createBucket(['Bucket' => $this->bucket]);
$this->testTimeout();
} catch (S3Exception $e) {
$logger->logException($e, [
'message' => 'Invalid remote storage.',
'level' => ILogger::DEBUG,
'app' => 'objectstore',
]);
throw new \Exception('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage());
}
if ($this->params['autocreate'] && !$this->connection->doesBucketExist($this->bucket)) {
$this->createNewBucket();
}

if ($this->params['autocreate'] && $this->sseUseBucketKey) {
$this->checkOrPutBucketKey();
}

// google cloud's s3 compatibility doesn't like the EncodingType parameter
Expand Down
16 changes: 12 additions & 4 deletions lib/private/Files/ObjectStore/S3ObjectTrait.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Files\ObjectStore;

Expand All @@ -43,6 +42,12 @@ trait S3ObjectTrait {
*/
abstract protected function getConnection();

/* compute configured encryption headers for put operations */
abstract protected function getSseKmsPutParameters();

/* compute configured encryption headers for get operations */
abstract protected function getSseKmsGetParameters();

/**
* @param string $urn the unified resource name used to identify the object
* @return resource stream with the read data
Expand All @@ -55,7 +60,7 @@ public function readObject($urn) {
'Bucket' => $this->bucket,
'Key' => $urn,
'Range' => 'bytes=' . $range,
]);
] + $this->getSseKmsGetParameters());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SseKmsPutParameters is placed under a config property, SseKmsGetParameters is not, is this right?

$request = \Aws\serialize($command);
$headers = [];
foreach ($request->getHeaders() as $key => $values) {
Expand Down Expand Up @@ -95,6 +100,7 @@ protected function writeSingle(string $urn, StreamInterface $stream, string $mim
'Body' => $stream,
'ACL' => 'private',
'ContentType' => $mimetype,
'params' => $this->getSseKmsPutParameters()
]);
}

Expand All @@ -114,7 +120,7 @@ protected function writeMultiPart(string $urn, StreamInterface $stream, string $
'part_size' => $this->uploadPartSize,
'params' => [
'ContentType' => $mimetype
],
] + $this->getSseKmsPutParameters(),
]);

try {
Expand Down Expand Up @@ -173,6 +179,8 @@ public function objectExists($urn) {
}

public function copyObject($from, $to) {
$this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to);
$this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [
'params' => $this->getSseKmsPutParameters(),
]);
}
}