From 0f7a93947c29f0ef53a5f18dedb7170bf3dff88e Mon Sep 17 00:00:00 2001 From: tsdicloud Date: Thu, 6 May 2021 14:57:11 +0000 Subject: [PATCH] Add S3 SSE KMS key and bucketked encryption --- .../Files/ObjectStore/S3ConnectionTrait.php | 185 +++++++++++++++--- .../Files/ObjectStore/S3ObjectTrait.php | 61 ++++-- 2 files changed, 197 insertions(+), 49 deletions(-) diff --git a/lib/private/Files/ObjectStore/S3ConnectionTrait.php b/lib/private/Files/ObjectStore/S3ConnectionTrait.php index d88ef0ac8e719..f676ece2ddaad 100644 --- a/lib/private/Files/ObjectStore/S3ConnectionTrait.php +++ b/lib/private/Files/ObjectStore/S3ConnectionTrait.php @@ -1,7 +1,6 @@ - * * @author Arthur Schiwon * @author Christoph Wurst * @author Florent @@ -10,7 +9,6 @@ * @author Roeland Jago Douma * @author S. Cat <33800996+sparrowjack63@users.noreply.github.com> * @author Stephen Cuppett - * * @license GNU AGPL version 3 or any later version * * This program is free software: you can redistribute it and/or modify @@ -25,15 +23,14 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - * */ namespace OC\Files\ObjectStore; use Aws\ClientResolver; use Aws\Credentials\CredentialProvider; -use Aws\Credentials\EcsCredentialProvider; use Aws\Credentials\Credentials; +use Aws\Credentials\EcsCredentialProvider; use Aws\Exception\CredentialsException; use Aws\S3\Exception\S3Exception; use Aws\S3\S3Client; @@ -60,25 +57,39 @@ trait S3ConnectionTrait { /** @var int */ protected $uploadPartSize; + /** @var string */ + protected $sseKmsKeyId; + + /** @var string */ + protected $sseKmsBucketKeyId; + protected $test; protected function parseParams($params) { if (empty($params['bucket'])) { - throw new \Exception("Bucket has to be configured."); + throw new \Exception('Bucket has to be configured.'); } - $this->id = 'amazon::' . $params['bucket']; + $this->id = 'amazon::'.$params['bucket']; $this->test = isset($params['test']); $this->bucket = $params['bucket']; $this->timeout = !isset($params['timeout']) ? 15 : $params['timeout']; $this->uploadPartSize = !isset($params['uploadPartSize']) ? 524288000 : $params['uploadPartSize']; $params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region']; - $params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname']; + $params['hostname'] = empty($params['hostname']) ? 's3.'.$params['region'].'.amazonaws.com' : $params['hostname']; if (!isset($params['port']) || $params['port'] === '') { $params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443; } - $params['verify_bucket_exists'] = empty($params['verify_bucket_exists']) ? true : $params['verify_bucket_exists']; + $params['autocreate'] = !isset($params['autocreate']) ? false : $params['autocreate']; + + // this avoid at least the hash lookups for each read/weite operation + if (isset($params['ssekmsbucketkeyid'])) { + $this->sseKmsBucketKeyId = $params['ssekmsbucketkeyid']; + } elseif (isset($params['ssekmskeyid'])) { + $this->sseKmsKeyId = $params['ssekmskeyid']; + } + $this->params = $params; } @@ -87,9 +98,129 @@ public function getBucket() { } /** - * Returns the connection + * Add the SSE KMS parameterdepending on the + * KMS encryption strategy (bucket, individual or + * no encryption) for object creations. + * + * @return array with encryption parameters + */ + public function getSseKmsPutParameters(): array { + if (empty($this->sseKmsBucketKeyId)) { + return [ + 'ServerSideEncryption' => 'aws:kms', + ]; + } elseif (empty($this->sseKmsKeyId)) { + return [ + 'ServerSideEncryption' => 'aws:kms', + 'SSEKMSKeyId' => $this->sseKmsKeyId, + ]; + } else { + return []; + } + } + + /** + * Add the SSE KMS parameter depending on the + * KMS encryption strategy (bucket, individual or + * no encryption) for object read. + * + * @return array with encryption parameters + */ + public function getSseKmsGetParameters(): array { + if (empty($this->sseKmsBucketKeyId)) { + return [ + 'ServerSideEncryption' => 'aws:kms', + 'SSEKMSKeyId' => $this->sseKmsBucketKeyId, + ]; + } elseif (empty($this->sseKmsKeyId)) { + return [ + 'ServerSideEncryption' => 'aws:kms', + 'SSEKMSKeyId' => $this->sseKmsKeyId, + ]; + } else { + return []; + } + } + + + /** + * Create the required bucket + * + * @throws \Exception if bucket creation fails + */ + protected function createNewBucket() { + $logger = \OC::$server->getLogger(); + try { + $logger->info('Bucket "'.$this->bucket.'" does not exist - creating it.', ['app' => 'objectstore']); + if (!$this->connection::isBucketDnsCompatible($this->bucket)) { + throw new \Exception('The bucket will not be created because the name is not dns compatible, please correct it: '.$this->bucket); + } + $this->connection->createBucket(['Bucket' => $this->bucket]); + $this->testTimeout(); + } catch (S3Exception $e) { + $logger->logException($e, [ + 'message' => 'Invalid remote storage.', + 'level' => ILogger::DEBUG, + 'app' => 'objectstore', + ]); + throw new \Exception('Creation of bucket "'.$this->bucket.'" failed. '.$e->getMessage()); + } + } + + /** + * Check bucket key consistency or put bucket key if missing + * This operation only works for bucket owner or with + * s3:GetEncryptionConfiguration/s3:PutEncryptionConfiguration permission + * + * We recommend to use autocreate only on initial setup and + * use an S3:user only with object operation permission and no bucket operation permissions + * later with autocreate=false + * + * @throws \Exception if bucket key config is inconsistent or if putting the key fails + */ + protected function checkOrPutBucketKey() { + $logger = \OC::$server->getLogger(); + + try { + $encrypt_state = $this->connection->getBucketEncryption([ + 'Bucket' => $this->bucket, + ]); + return; + } catch (S3Exception $e) { + try { + $logger->info('Bucket key for "'.$this->bucket.'" is not set - adding it.', ['app' => 'objectstore']); + $this->connection->putBucketEncryption([ + 'Bucket' => $this->bucket , + 'ServerSideEncryptionConfiguration' => [ + 'Rules' => [ + [ + 'ApplyServerSideEncryptionByDefault' => [ + 'KMSMasterKeyID' => $this->sseKmsBucketKeyId, + 'SSEAlgorithm' => 'aws:kms', + ], + 'BucketKeyEnabled' => true, + ], + ], + ], + ]); + $this->testTimeout(); + } catch (S3Exception $e) { + $logger->logException($e, [ + 'message' => 'Bucket key problem.', + 'level' => ILogger::DEBUG, + 'app' => 'objectstore', + ]); + throw new \Exception('Putting configured bucket key to "'.$this->bucket.'" failed. '.$e->getMessage()); + } + } + } + + + /** + * Returns the connection. * * @return S3Client connected client + * * @throws \Exception if connection could not be made */ public function getConnection() { @@ -98,7 +229,7 @@ public function getConnection() { } $scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https'; - $base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/'; + $base_url = $scheme.'://'.$this->params['hostname'].':'.$this->params['port'].'/'; // Adding explicit credential provider to the beginning chain. // Including environment variables and IAM instance profiles. @@ -132,27 +263,16 @@ public function getConnection() { if (!$this->connection::isBucketDnsCompatible($this->bucket)) { $logger = \OC::$server->getLogger(); - $logger->debug('Bucket "' . $this->bucket . '" This bucket name is not dns compatible, it may contain invalid characters.', - ['app' => 'objectstore']); + $logger->debug('Bucket "'.$this->bucket.'" This bucket name is not dns compatible, it may contain invalid characters.', + ['app' => 'objectstore']); } - if ($this->params['verify_bucket_exists'] && !$this->connection->doesBucketExist($this->bucket)) { - $logger = \OC::$server->getLogger(); - try { - $logger->info('Bucket "' . $this->bucket . '" does not exist - creating it.', ['app' => 'objectstore']); - if (!$this->connection::isBucketDnsCompatible($this->bucket)) { - throw new \Exception("The bucket will not be created because the name is not dns compatible, please correct it: " . $this->bucket); - } - $this->connection->createBucket(['Bucket' => $this->bucket]); - $this->testTimeout(); - } catch (S3Exception $e) { - $logger->logException($e, [ - 'message' => 'Invalid remote storage.', - 'level' => ILogger::DEBUG, - 'app' => 'objectstore', - ]); - throw new \Exception('Creation of bucket "' . $this->bucket . '" failed. ' . $e->getMessage()); - } + if ($this->params['autocreate'] && !$this->connection->doesBucketExist($this->bucket)) { + $this->createNewBucket(); + } + + if ($this->params['autocreate'] && isset($this->params['ssekmsbucketkeyid'])) { + $this->checkOrPutBucketKey(); } // google cloud's s3 compatibility doesn't like the EncodingType parameter @@ -164,7 +284,7 @@ public function getConnection() { } /** - * when running the tests wait to let the buckets catch up + * when running the tests wait to let the buckets catch up. */ private function testTimeout() { if ($this->test) { @@ -183,9 +303,9 @@ public static function legacySignatureProvider($version, $service, $region) { } /** - * This function creates a credential provider based on user parameter file + * This function creates a credential provider based on user parameter file. */ - protected function paramCredentialProvider() : callable { + protected function paramCredentialProvider(): callable { return function () { $key = empty($this->params['key']) ? null : $this->params['key']; $secret = empty($this->params['secret']) ? null : $this->params['secret']; @@ -197,6 +317,7 @@ protected function paramCredentialProvider() : callable { } $msg = 'Could not find parameters set for credentials in config file.'; + return new RejectedPromise(new CredentialsException($msg)); }; } diff --git a/lib/private/Files/ObjectStore/S3ObjectTrait.php b/lib/private/Files/ObjectStore/S3ObjectTrait.php index 250f8fd1edd01..8b8cc444ad99e 100644 --- a/lib/private/Files/ObjectStore/S3ObjectTrait.php +++ b/lib/private/Files/ObjectStore/S3ObjectTrait.php @@ -1,13 +1,11 @@ - * * @author Christoph Wurst * @author Florent * @author Morris Jobke * @author Robin Appelman * @author Roeland Jago Douma - * * @license GNU AGPL version 3 or any later version * * This program is free software: you can redistribute it and/or modify @@ -22,7 +20,6 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - * */ namespace OC\Files\ObjectStore; @@ -36,26 +33,37 @@ trait S3ObjectTrait { /** - * Returns the connection + * Returns the connection. * * @return S3Client connected client + * * @throws \Exception if connection could not be made */ abstract protected function getConnection(); + /* compute configured encryption headers for put operations */ + abstract protected function getSseKmsPutParameters(); + + /* compute configured encryption headers for get operations */ + abstract protected function getSseKmsGetParameters(); + /** * @param string $urn the unified resource name used to identify the object + * * @return resource stream with the read data + * * @throws \Exception when something goes wrong, message will be logged + * * @since 7.0.0 */ public function readObject($urn) { return SeekableHttpStream::open(function ($range) use ($urn) { - $command = $this->getConnection()->getCommand('GetObject', [ + $s3params = [ 'Bucket' => $this->bucket, 'Key' => $urn, - 'Range' => 'bytes=' . $range, - ]); + 'Range' => 'bytes='.$range, + ] + $this->getSseKmsGetParameters(); + $command = $this->getConnection()->getCommand('GetObject', $s3params); $request = \Aws\serialize($command); $headers = []; foreach ($request->getHeaders() as $key => $values) { @@ -71,49 +79,63 @@ public function readObject($urn) { ]; $context = stream_context_create($opts); + return fopen($request->getUri(), 'r', false, $context); }); } /** - * @param string $urn the unified resource name used to identify the object + * @param string $urn the unified resource name used to identify the object * @param resource $stream stream with the data to write - * @param string|null $mimetype the mimetype to set for the remove object @since 22.0.0 + * * @throws \Exception when something goes wrong, message will be logged + * * @since 7.0.0 */ - public function writeObject($urn, $stream, string $mimetype = null) { + public function writeObject($urn, $stream) { $count = 0; $countStream = CallbackWrapper::wrap($stream, function ($read) use (&$count) { $count += $read; }); - $uploader = new MultipartUploader($this->getConnection(), $countStream, [ + $s3params = [ 'bucket' => $this->bucket, 'key' => $urn, 'part_size' => $this->uploadPartSize, - 'params' => [ - 'ContentType' => $mimetype - ] - ]); + 'params' => $this->getSseKmsPutParameters(), + ]; + $uploader = new MultipartUploader($this->getConnection(), $countStream, $s3params); try { $uploader->upload(); } catch (S3MultipartUploadException $e) { // This is an empty file so just touch it then if ($count === 0 && feof($countStream)) { - $uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, ''); + $s3params = [ + 'params' => $this->getSseKmsPutParameters(), + ]; + $uploader = new ObjectUploader($this->getConnection(), $this->bucket, $urn, '', 'private', $s3params); $uploader->upload(); } else { throw $e; } + } finally { + // this handles [S3] fclose(): supplied resource is not a valid stream resource #23373 + // see https://stackoverflow.com/questions/11247507/fclose-18-is-not-a-valid-stream-resource/11247555 + // which also recommends the solution + if (is_resource($countStream)) { + fclose($countStream); + } } } /** * @param string $urn the unified resource name used to identify the object + * * @return void + * * @throws \Exception when something goes wrong, message will be logged + * * @since 7.0.0 */ public function deleteObject($urn) { @@ -127,7 +149,12 @@ public function objectExists($urn) { return $this->getConnection()->doesObjectExist($this->bucket, $urn); } + /** + * S3 copy command with SSE KMS key handling. + */ public function copyObject($from, $to) { - $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to); + $this->getConnection()->copy($this->getBucket(), $from, $this->getBucket(), $to, 'private', [ + 'params' => $this->getSseKmsPutParameters(), + ]); } }