S3ConnectionTrait.php

<?php
/**
 * @copyright Copyright (c) 2016 Robin Appelman <robin@icewind.nl>
 * @author Arthur Schiwon <blizzz@arthur-schiwon.de>
 * @author Christoph Wurst <christoph@winzerhof-wurst.at>
 * @author Florent <florent@coppint.com>
 * @author Morris Jobke <hey@morrisjobke.de>
 * @author Robin Appelman <robin@icewind.nl>
 * @author Roeland Jago Douma <roeland@famdouma.nl>
 * @author S. Cat <33800996+sparrowjack63@users.noreply.github.com>
 * @author Stephen Cuppett <steve@cuppett.com>
 * @license GNU AGPL version 3 or any later version
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

namespace OC\Files\ObjectStore;

use Aws\ClientResolver;
use Aws\Credentials\CredentialProvider;
use Aws\Credentials\Credentials;
use Aws\Credentials\EcsCredentialProvider;
use Aws\Exception\CredentialsException;
use Aws\S3\Exception\S3Exception;
use Aws\S3\S3Client;
use GuzzleHttp\Promise;
use GuzzleHttp\Promise\RejectedPromise;
use OCP\ILogger;


trait S3ConnectionTrait {
	/** @var array */
	protected $params;

	/** @var S3Client */
	protected $connection;

	/** @var string */
	protected $id;

	/** @var string */
	protected $bucket;

	/** @var int */
	protected $timeout;

	/** @var int */
	protected $uploadPartSize;

	/** @var string */
	protected $sseKmsKeyId;

	/** @var bool */
	protected $sseUseBucketKey;

	protected $test;

	protected function parseParams($params) {
		if (empty($params['bucket'])) {
			throw new \Exception('Bucket has to be configured.');
		}

		$this->id = 'amazon::' . $params['bucket'];

		$this->test = isset($params['test']);
		$this->bucket = $params['bucket'];
		$this->timeout = !isset($params['timeout']) ? 15 : $params['timeout'];
		$this->uploadPartSize = !isset($params['uploadPartSize']) ? 524288000 : $params['uploadPartSize'];
		$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
		$params['hostname'] = empty($params['hostname']) ? 's3.' . $params['region'] . '.amazonaws.com' : $params['hostname'];
		if (!isset($params['port']) || $params['port'] === '') {
			$params['port'] = (isset($params['use_ssl']) && $params['use_ssl'] === false) ? 80 : 443;
		}
		$params['autocreate'] = !isset($params['autocreate']) ? false : $params['autocreate'];

		// this avoid at least the hash lookups for each read/weite operation
		if (isset($params['ssekmskeyid'])) {
			$this->sseKmsKeyId = $params['ssekmskeyid'];
		}
		$this->sseUseBucketKey = (isset($params['sseusebucketkey'])) ? $params['sseusebucketkey'] : false;

		$this->params = $params;
	}

	public function getBucket() {
		return $this->bucket;
	}

	/**
	 * Add the SSE KMS parameterdepending on the
	 * KMS encryption strategy (bucket, individual or
	 * no encryption) for object creations.
	 *
	 * @return array with encryption parameters
	 */
	public function getSseKmsPutParameters(): array {
		if ($this->sseUseBucketKey) {
			return [
				'ServerSideEncryption' => 'aws:kms',
				'BucketKeyEnabled' => true,
			];
		} elseif (!empty($this->sseKmsKeyId)) {
			return [
				'ServerSideEncryption' => 'aws:kms',
				'BucketKeyEnabled' => false,
				'SSEKMSKeyId' => $this->sseKmsKeyId,
			];
		} else {
			return [];
		}
	}

	/**
	 * Add the SSE KMS parameter depending on the
	 * KMS encryption strategy (bucket, individual or
	 * no encryption) for object read.
	 *
	 * @return array with encryption parameters
	 */
	public function getSseKmsGetParameters(): array {
		if (($this->sseUseBucketKey) || !empty($this->sseKmsKeyId)) {
			return [
				'ServerSideEncryption' => 'aws:kms',
			];
		} else {
			return [];
		}
	}

	/**
	 * Create the required bucket
	 *
	 * @throws \Exception if bucket creation fails
	 */
	protected function createNewBucket() {
		$logger = \OC::$server->getLogger();
		try {
			$logger->info('Bucket "'.$this->bucket.'" does not exist - creating it.', ['app' => 'objectstore']);
			if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
				throw new \Exception('The bucket will not be created because the name is not dns compatible, please correct it: '.$this->bucket);
			}
			$this->connection->createBucket(['Bucket' => $this->bucket]);
			$this->testTimeout();
		} catch (S3Exception $e) {
			$logger->logException($e, [
				'message' => 'Invalid remote storage.',
				'level' => ILogger::DEBUG,
				'app' => 'objectstore',
			]);
			throw new \Exception('Creation of bucket "'.$this->bucket.'" failed. '.$e->getMessage());
		}
	}

	/**
	 * Check bucket key consistency or put bucket key if missing
	 * This operation only works for bucket owner or with
	 * s3:GetEncryptionConfiguration/s3:PutEncryptionConfiguration permission
	 *
	 * We recommend to use autocreate only on initial setup and
	 * use an S3:user only with object operation permission and no bucket operation permissions
	 * later with autocreate=false
	 *
	 * @throws \Exception if bucket key config is inconsistent or if putting the key fails
	 */
	protected function checkOrPutBucketKey() {
		$logger = \OC::$server->getLogger();

		try {
			$encrypt_state = $this->connection->getBucketEncryption([
				'Bucket' => $this->bucket,
			]);
		} catch (S3Exception $e) {
			try {
				$logger->info('Bucket key for "'.$this->bucket.'" is not set - adding it.', ['app' => 'objectstore']);
				$this->connection->putBucketEncryption([
					'Bucket' => $this->bucket,
					'ServerSideEncryptionConfiguration' => [
						'Rules' => [
							[
								'ApplyServerSideEncryptionByDefault' => [
									'SSEAlgorithm' => 'aws:kms',
									'KMSMasterKeyID' => $this->sseKmsKeyId,
								],
							],
						],
					],
				]);
				$this->testTimeout();
			} catch (S3Exception $e) {
				$logger->logException($e, [
					'message' => 'Bucket key problem.',
					'level' => ILogger::DEBUG,
					'app' => 'objectstore',
				]);
				throw new \Exception('Putting configured bucket key to "'.$this->bucket.'" failed. '.$e->getMessage());
			}
		}
	}


	/**
	 * Returns the connection.
	 *
	 * @return S3Client connected client
	 *
	 * @throws \Exception if connection could not be made
	 */
	public function getConnection() {
		if (!is_null($this->connection)) {
			return $this->connection;
		}

		$scheme = (isset($this->params['use_ssl']) && $this->params['use_ssl'] === false) ? 'http' : 'https';
		$base_url = $scheme.'://'.$this->params['hostname'].':'.$this->params['port'].'/';

		// Adding explicit credential provider to the beginning chain.
		// Including environment variables and IAM instance profiles.
		$provider = CredentialProvider::memoize(
			CredentialProvider::chain(
				$this->paramCredentialProvider(),
				CredentialProvider::env(),
				CredentialProvider::assumeRoleWithWebIdentityCredentialProvider(),
				!empty(getenv(EcsCredentialProvider::ENV_URI))
					? CredentialProvider::ecsCredentials()
					: CredentialProvider::instanceProfile()
			)
		);

		$options = [
			'version' => isset($this->params['version']) ? $this->params['version'] : 'latest',
			'credentials' => $provider,
			'endpoint' => $base_url,
			'region' => $this->params['region'],
			'use_path_style_endpoint' => isset($this->params['use_path_style']) ? $this->params['use_path_style'] : false,
			'signature_provider' => \Aws\or_chain([self::class, 'legacySignatureProvider'], ClientResolver::_default_signature_provider()),
			'csm' => false,
			// 'debug'   => true, // to debug S3 communication
		];
		if (isset($this->params['proxy'])) {
			$options['request.options'] = ['proxy' => $this->params['proxy']];
		}
		if (isset($this->params['legacy_auth']) && $this->params['legacy_auth']) {
			$options['signature_version'] = 'v2';
		}
		$this->connection = new S3Client($options);

		if (!$this->connection::isBucketDnsCompatible($this->bucket)) {
			$logger = \OC::$server->getLogger();
			$logger->debug('Bucket "' . $this->bucket.'" This bucket name is not dns compatible, it may contain invalid characters.',
					['app' => 'objectstore']);
		}

		if ($this->params['autocreate'] && !$this->connection->doesBucketExist($this->bucket)) {
			$this->createNewBucket();
		}

		if ($this->params['autocreate'] && $this->sseUseBucketKey) {
			$this->checkOrPutBucketKey();
		}

		// google cloud's s3 compatibility doesn't like the EncodingType parameter
		if (strpos($base_url, 'storage.googleapis.com')) {
			$this->connection->getHandlerList()->remove('s3.auto_encode');
		}

		return $this->connection;
	}

	/**
	 * when running the tests wait to let the buckets catch up.
	 */
	private function testTimeout() {
		if ($this->test) {
			sleep($this->timeout);
		}
	}

	public static function legacySignatureProvider($version, $service, $region) {
		switch ($version) {
			case 'v2':
			case 's3':
				return new S3Signature();
			default:
				return null;
		}
	}

	/**
	 * This function creates a credential provider based on user parameter file.
	 */
	protected function paramCredentialProvider(): callable {
		return function () {
			$key = empty($this->params['key']) ? null : $this->params['key'];
			$secret = empty($this->params['secret']) ? null : $this->params['secret'];

			if ($key && $secret) {
				return Promise\promise_for(
					new Credentials($key, $secret)
				);
			}

			$msg = 'Could not find parameters set for credentials in config file.';

			return new RejectedPromise(new CredentialsException($msg));
		};
	}
}