From 8251362828582e2f810ae166d7b77a62b3cd70ac Mon Sep 17 00:00:00 2001 From: Matt Houglum Date: Fri, 9 Aug 2019 00:19:48 +0000 Subject: [PATCH] Add virtual hosted-style support for signurl gen. Adds new methods to produce a SignUrlOption that results in generated signed URLs using virtual-hosted-style URLs (i.e. mybucket.storage.googleapis.com instead of storage.googleapis.com/mybucket). One option allows specifying the virtual hostname explicitly (for the case where someone might have a custom subdomain, with a bucket of the same name, CNAME'd to c.storage.googleapis.com), while the other will implicitly construct the hostname using the bucket from the passed-in BlobInfo. Addresses part of https://issuetracker.google.com/issues/130190655. --- .../com/google/cloud/storage/Storage.java | 34 +++++- .../com/google/cloud/storage/StorageImpl.java | 114 +++++++++++++++--- 2 files changed, 128 insertions(+), 20 deletions(-) diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java index 8b3771d2c2cf..311dc7cdb53a 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java @@ -1035,7 +1035,8 @@ enum Option { EXT_HEADERS, SERVICE_ACCOUNT_CRED, SIGNATURE_VERSION, - HOST_NAME + HOST_NAME, + VIRTUAL_HOST_NAME } enum SignatureVersion { @@ -1122,11 +1123,40 @@ public static SignUrlOption signWith(ServiceAccountSigner signer) { /** * Use a different host name than the default host name 'https://storage.googleapis.com'. This - * must also include the scheme component of the URI. + * must also include the scheme component of the URI. Note that this cannot be used alongside + * {@code withVirtualHostName()}. */ public static SignUrlOption withHostName(String hostName) { return new SignUrlOption(Option.HOST_NAME, hostName); } + + /** + * Use a virtual hosted-style hostname, which includes the bucket in the host portion of the URI + * rather than the path, e.g. 'https://mybucket.storage.googleapis.com'. This must also include + * the scheme component of the URI. Note that this cannot be used alongside {@code + * withHostName()}. For V4 signing, this also sets the "host" header in the canonicalized + * extension headers to the specified value, minus the "http[s]://", unless that header is + * supplied via the {@code withExtHeaders()} method. + * + * @see Request Endpoints + */ + public static SignUrlOption withVirtualHostName(String virtualHostName) { + return new SignUrlOption(Option.VIRTUAL_HOST_NAME, virtualHostName); + } + + /** + * Use a virtual hosted-style hostname, which includes the bucket in the host portion of the URI + * rather than the path, e.g. 'https://mybucket.storage.googleapis.com'. The bucket name will be + * obtained from the resource passed in. Note that this cannot be used alongside {@code + * withHostName()}. For V4 signing, this also sets the "host" header in the canonicalized + * extension headers to the virtual hosted-style host, unless that header is supplied via the + * {@code withExtHeaders()} method. + * + * @see Request Endpoints + */ + public static SignUrlOption withVirtualHostName() { + return new SignUrlOption(Option.VIRTUAL_HOST_NAME, ""); + } } /** diff --git a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 8b6ccf3b16b3..b5673bf42976 100644 --- a/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-clients/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -88,7 +88,9 @@ final class StorageImpl extends BaseService implements Storage { private static final String EMPTY_BYTE_ARRAY_CRC32C = "AAAAAA=="; private static final String PATH_DELIMITER = "/"; /** Signed URLs are only supported through the GCS XML API endpoint. */ - private static final String STORAGE_XML_HOST_NAME = "https://storage.googleapis.com"; + private static final String STORAGE_XML_URI_SCHEME = "https"; + + private static final String STORAGE_XML_URI_HOST_NAME = "storage.googleapis.com"; private static final Function, Boolean> DELETE_FUNCTION = new Function, Boolean>() { @@ -635,6 +637,9 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio optionMap.put(option.getOption(), option.getValue()); } + boolean isV2 = + SignUrlOption.SignatureVersion.V2.equals( + optionMap.get(SignUrlOption.Option.SIGNATURE_VERSION)); boolean isV4 = SignUrlOption.SignatureVersion.V4.equals( optionMap.get(SignUrlOption.Option.SIGNATURE_VERSION)); @@ -655,14 +660,12 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio getOptions().getClock().millisTime() + unit.toMillis(duration), TimeUnit.MILLISECONDS); - String storageXmlHostName = - optionMap.get(SignUrlOption.Option.HOST_NAME) != null - ? (String) optionMap.get(SignUrlOption.Option.HOST_NAME) - : STORAGE_XML_HOST_NAME; + checkArgument( + !(optionMap.get(SignUrlOption.Option.VIRTUAL_HOST_NAME) != null + && optionMap.get(SignUrlOption.Option.HOST_NAME) != null), + "Cannot specify both the VIRTUAL_HOST_NAME and HOST_NAME SignUrlOptions together."); - // The bucket name itself should never contain a forward slash. However, parts already existed - // in the code to check for this, so we remove the forward slashes to be safe here. - String bucketName = CharMatcher.anyOf(PATH_DELIMITER).trimFrom(blobInfo.getBucket()); + String bucketName = slashlessBucketNameFromBlobInfo(blobInfo); String escapedBlobName = ""; if (!Strings.isNullOrEmpty(blobInfo.getName())) { escapedBlobName = @@ -672,12 +675,35 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio .replace(";", "%3B"); } - String stPath = constructResourceUriPath(bucketName, escapedBlobName); + String storageXmlHostName; + boolean useBucketInPath = true; + if (optionMap.get(SignUrlOption.Option.VIRTUAL_HOST_NAME) != null) { + // In virtual hosted-style endpoints, the bucket is included in the host portion of the URI + // instead of in the path. + useBucketInPath = false; + storageXmlHostName = + virtualHostFromOptionValue( + (String) optionMap.get(SignUrlOption.Option.VIRTUAL_HOST_NAME), bucketName); + } else if (optionMap.get(SignUrlOption.Option.HOST_NAME) != null) { + storageXmlHostName = (String) optionMap.get(SignUrlOption.Option.HOST_NAME); + } else { + storageXmlHostName = STORAGE_XML_URI_SCHEME + "://" + STORAGE_XML_URI_HOST_NAME; + } + + String stPath = + useBucketInPath + ? constructResourceUriPath(bucketName, escapedBlobName, optionMap) + : constructResourceUriPath("", escapedBlobName, optionMap); URI path = URI.create(stPath); + // For V2 signing, even if we don't specify the bucket in the URI path, we still need the + // canonical resource string that we'll sign to include the bucket. + URI pathForSigning = + isV2 ? URI.create(constructResourceUriPath(bucketName, escapedBlobName, optionMap)) : path; try { SignatureInfo signatureInfo = - buildSignatureInfo(optionMap, blobInfo, expiration, path, credentials.getAccount()); + buildSignatureInfo( + optionMap, blobInfo, expiration, pathForSigning, credentials.getAccount()); String unsignedPayload = signatureInfo.constructUnsignedPayload(); byte[] signatureBytes = credentials.sign(unsignedPayload.getBytes(UTF_8)); StringBuilder stBuilder = new StringBuilder(); @@ -705,10 +731,31 @@ public URL signUrl(BlobInfo blobInfo, long duration, TimeUnit unit, SignUrlOptio } } - private String constructResourceUriPath(String slashlessBucketName, String escapedBlobName) { + private String constructResourceUriPath( + String slashlessBucketName, + String escapedBlobName, + EnumMap optionMap) { + if (Strings.isNullOrEmpty(slashlessBucketName)) { + if (Strings.isNullOrEmpty(escapedBlobName)) { + return PATH_DELIMITER; + } + if (escapedBlobName.startsWith(PATH_DELIMITER)) { + return escapedBlobName; + } + return PATH_DELIMITER + escapedBlobName; + } + StringBuilder pathBuilder = new StringBuilder(); pathBuilder.append(PATH_DELIMITER).append(slashlessBucketName); if (Strings.isNullOrEmpty(escapedBlobName)) { + boolean isV2 = + SignUrlOption.SignatureVersion.V2.equals( + optionMap.get(SignUrlOption.Option.SIGNATURE_VERSION)); + // If using virtual-hosted style URLs with V2 signing, the path string for a bucket resource + // must end with a forward slash. + if (optionMap.get(SignUrlOption.Option.VIRTUAL_HOST_NAME) != null && isV2) { + pathBuilder.append(PATH_DELIMITER); + } return pathBuilder.toString(); } if (!escapedBlobName.startsWith(PATH_DELIMITER)) { @@ -760,14 +807,45 @@ private SignatureInfo buildSignatureInfo( signatureInfoBuilder.setTimestamp(getOptions().getClock().millisTime()); - @SuppressWarnings("unchecked") - Map extHeaders = - (Map) - (optionMap.containsKey(SignUrlOption.Option.EXT_HEADERS) - ? (Map) optionMap.get(SignUrlOption.Option.EXT_HEADERS) - : Collections.emptyMap()); + ImmutableMap.Builder extHeaders = new ImmutableMap.Builder(); + + boolean isV4 = + SignUrlOption.SignatureVersion.V4.equals( + optionMap.get(SignUrlOption.Option.SIGNATURE_VERSION)); + // V2 signing requires that the header not include the bucket, but V4 signing requires that + // the host name used in the URI must match the "host" header. + boolean setHostHeaderToVirtualHost = + optionMap.containsKey(SignUrlOption.Option.VIRTUAL_HOST_NAME) && isV4; + // Add this host first if needed, allowing it to be overridden in the EXT_HEADERS option below. + if (setHostHeaderToVirtualHost) { + String vhost = + virtualHostFromOptionValue( + (String) optionMap.get(SignUrlOption.Option.VIRTUAL_HOST_NAME), + slashlessBucketNameFromBlobInfo(blobInfo)); + vhost = vhost.replaceFirst("http(s)?://", ""); + extHeaders.put("host", vhost); + } - return signatureInfoBuilder.setCanonicalizedExtensionHeaders(extHeaders).build(); + if (optionMap.containsKey(SignUrlOption.Option.EXT_HEADERS)) { + extHeaders.putAll((Map) optionMap.get(SignUrlOption.Option.EXT_HEADERS)); + } + + return signatureInfoBuilder + .setCanonicalizedExtensionHeaders((Map) extHeaders.build()) + .build(); + } + + private String slashlessBucketNameFromBlobInfo(BlobInfo blobInfo) { + // The bucket name itself should never contain a forward slash. However, parts already existed + // in the code to check for this, so we remove the forward slashes to be safe here. + return CharMatcher.anyOf(PATH_DELIMITER).trimFrom(blobInfo.getBucket()); + } + + private String virtualHostFromOptionValue(String vhostOptionValue, String bucketName) { + if (Strings.isNullOrEmpty(vhostOptionValue)) { + return STORAGE_XML_URI_SCHEME + "://" + bucketName + "." + STORAGE_XML_URI_HOST_NAME; + } + return vhostOptionValue; } @Override