Upgrade GCS SDK to 1.104.0 (elastic#52839)

Upgrading the GCS SDK to the most recent version.
Adjusting (i.e. improving) the REST mock accordingly.
This should significantly boost performance by pulling in
googleapis/java-core#86 in some cases.

plugins/repository-gcs/build.gradle

@@ -25,10 +25,10 @@ esplugin {
 }
 
 dependencies {
-  compile 'com.google.cloud:google-cloud-storage:1.77.0'
-  compile 'com.google.cloud:google-cloud-core:1.77.0'
+  compile 'com.google.cloud:google-cloud-storage:1.104.0'
+  compile 'com.google.cloud:google-cloud-core:1.92.5'
   compile 'com.google.guava:guava:26.0-jre'
-  compile 'com.google.http-client:google-http-client:1.30.1'
+  compile 'com.google.http-client:google-http-client:1.34.2'
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
@@ -37,23 +37,23 @@ dependencies {
   compile 'com.google.api:api-common:1.8.1'
   compile 'com.google.api:gax:1.45.0'
   compile 'org.threeten:threetenbp:1.3.3'
-  compile 'com.google.protobuf:protobuf-java-util:3.7.1'
-  compile 'com.google.protobuf:protobuf-java:3.7.1'
+  compile 'com.google.protobuf:protobuf-java-util:3.11.3'
+  compile 'com.google.protobuf:protobuf-java:3.11.3'
   compile 'com.google.code.gson:gson:2.7'
   compile 'com.google.api.grpc:proto-google-common-protos:1.16.0'
   compile 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
   compile 'com.google.cloud:google-cloud-core-http:1.77.0'
-  compile 'com.google.auth:google-auth-library-credentials:0.16.1'
-  compile 'com.google.auth:google-auth-library-oauth2-http:0.16.1'
+  compile 'com.google.auth:google-auth-library-credentials:0.20.0'
+  compile 'com.google.auth:google-auth-library-oauth2-http:0.20.0'
   compile 'com.google.oauth-client:google-oauth-client:1.28.0'
-  compile 'com.google.api-client:google-api-client:1.28.0'
+  compile 'com.google.api-client:google-api-client:1.30.8'
   compile 'com.google.http-client:google-http-client-appengine:1.29.2'
   compile 'com.google.http-client:google-http-client-jackson2:1.29.2'
   compile 'com.google.api:gax-httpjson:0.62.0'
   compile 'io.grpc:grpc-context:1.12.0'
   compile 'io.opencensus:opencensus-api:0.18.0'
   compile 'io.opencensus:opencensus-contrib-http-util:0.18.0'
-  compile 'com.google.apis:google-api-services-storage:v1-rev20190426-1.28.0'
+  compile 'com.google.apis:google-api-services-storage:v1-rev20191011-1.30.3'
 
   testCompile project(':test:fixtures:gcs-fixture')
 }
@@ -75,6 +75,9 @@ thirdPartyAudit {
     'com.google.protobuf.UnsafeUtil$1',
     'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
     'com.google.protobuf.UnsafeUtil$MemoryAccessor',
+    'com.google.protobuf.MessageSchema',
+    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
     'com.google.common.cache.Striped64',
     'com.google.common.cache.Striped64$1',
     'com.google.common.cache.Striped64$Cell',

plugins/repository-gcs/licenses/google-api-client-1.30.8.jar.sha1

@@ -0,0 +1 @@
+b8664bfb9287f269228a4bbc36988174d4f311eb

plugins/repository-gcs/licenses/google-api-services-storage-v1-rev20191011-1.30.3.jar.sha1

@@ -0,0 +1 @@
+53fc173da380f9b65caad69912223db128dbf7ac

plugins/repository-gcs/licenses/google-auth-library-credentials-0.20.0.jar.sha1

@@ -0,0 +1 @@
+87a91a373e64ba5c3cdf8cc5cf54b189dd1492f8

plugins/repository-gcs/licenses/google-auth-library-oauth2-http-0.20.0.jar.sha1

@@ -0,0 +1 @@
+f33d4d6c91a68826816606a2208990eea93fcb2a

plugins/repository-gcs/licenses/google-cloud-core-1.92.5.jar.sha1

@@ -0,0 +1 @@
+97ca5c35f34ee67dde8f2d4e9b940f0a96393c66

plugins/repository-gcs/licenses/google-cloud-storage-1.104.0.jar.sha1

@@ -0,0 +1 @@
+8a083661f3732b21c0addd4fe56fd3f05a9c5a84

plugins/repository-gcs/licenses/google-http-client-1.34.2.jar.sha1

@@ -0,0 +1 @@
+58b4c500b5ce58271f1a6c5f6767aa8721158ea4

plugins/repository-gcs/licenses/protobuf-java-3.11.3.jar.sha1

@@ -0,0 +1 @@
+df12be70b968e32442821a2cfdc3cede5a42dec5

plugins/repository-gcs/licenses/protobuf-java-util-3.11.3.jar.sha1

@@ -0,0 +1 @@
+fd4ba2dfeb1b010eb20ca27e65fbfb74fbbdcdb9

plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java

@@ -132,6 +132,8 @@ StorageOptions createStorageOptions(final GoogleCloudStorageClientSettings clien
                                                 final HttpTransportOptions httpTransportOptions) {
                 StorageOptions options = super.createStorageOptions(clientSettings, httpTransportOptions);
                 return options.toBuilder()
+                    .setHost(options.getHost())
+                    .setCredentials(options.getCredentials())
                     .setRetrySettings(RetrySettings.newBuilder()
                         .setTotalTimeout(options.getRetrySettings().getTotalTimeout())
                         .setInitialRetryDelay(Duration.ofMillis(10L))
@@ -331,7 +333,7 @@ public void testWriteBlobWithReadTimeouts() {
 
     public void testWriteLargeBlob() throws IOException {
         // See {@link BaseWriteChannel#DEFAULT_CHUNK_SIZE}
-        final int defaultChunkSize = 8 * 256 * 1024;
+        final int defaultChunkSize = 60 * 256 * 1024;
         final int nbChunks = randomIntBetween(3, 5);
         final int lastChunkSize = randomIntBetween(1, defaultChunkSize - 1);
         final int totalChunks = nbChunks + 1;

plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java

@@ -202,6 +202,8 @@ StorageOptions createStorageOptions(final GoogleCloudStorageClientSettings clien
                                                     final HttpTransportOptions httpTransportOptions) {
                     StorageOptions options = super.createStorageOptions(clientSettings, httpTransportOptions);
                     return options.toBuilder()
+                        .setHost(options.getHost())
+                        .setCredentials(options.getCredentials())
                         .setRetrySettings(RetrySettings.newBuilder()
                             .setTotalTimeout(options.getRetrySettings().getTotalTimeout())
                             .setInitialRetryDelay(Duration.ofMillis(10L))
@@ -269,7 +271,7 @@ protected String requestUniqueId(HttpExchange exchange) {
             }
 
             final String range = exchange.getRequestHeaders().getFirst("Content-Range");
-            return exchange.getRemoteAddress().toString()
+            return exchange.getRemoteAddress().getHostString()
                 + " " + exchange.getRequestMethod()
                 + " " + exchange.getRequestURI()
                 + (range != null ?  " " + range :  "");

test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java

@@ -298,9 +298,10 @@ public static Optional<Tuple<String, BytesReference>> parseMultipartRequestBody(
                 skippedEmptyLine = markAndContinue && endPos == startPos;
                 startPos = endPos;
             } else {
-                // removes the trailing end "\r\n--__END_OF_PART__--\r\n" which is 23 bytes long
-                int len = fullRequestBody.length() - startPos - 23;
-                content = Tuple.tuple(name, fullRequestBody.slice(startPos, len));
+                while (isEndOfPart(fullRequestBody, endPos) == false) {
+                    endPos = fullRequestBody.indexOf((byte) '\r', endPos + 1);
+                }
+                content = Tuple.tuple(name, fullRequestBody.slice(startPos, endPos - startPos));
                 break;
             }
         }
@@ -312,6 +313,18 @@ public static Optional<Tuple<String, BytesReference>> parseMultipartRequestBody(
         return Optional.ofNullable(content);
     }
 
+    private static final byte[] END_OF_PARTS_MARKER = "\r\n--__END_OF_PART__".getBytes(UTF_8);
+
+    private static boolean isEndOfPart(BytesReference fullRequestBody, int endPos) {
+        for (int i = 0; i < END_OF_PARTS_MARKER.length; i++) {
+            final byte b = END_OF_PARTS_MARKER[i];
+            if (fullRequestBody.get(endPos + i) != b) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     private static final Pattern PATTERN_CONTENT_RANGE = Pattern.compile("bytes ([^/]*)/([0-9\\*]*)");
     private static final Pattern PATTERN_CONTENT_RANGE_BYTES = Pattern.compile("([0-9]*)-([0-9]*)");
 

x-pack/snapshot-tool/build.gradle

@@ -30,10 +30,10 @@ dependencies {
   compile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 
   // GCS dependencies
-  compile 'com.google.cloud:google-cloud-storage:1.77.0'
-  compile 'com.google.cloud:google-cloud-core:1.77.0'
+  compile 'com.google.cloud:google-cloud-storage:1.104.0'
+  compile 'com.google.cloud:google-cloud-core:1.92.5'
   compile 'com.google.guava:guava:26.0-jre'
-  compile 'com.google.http-client:google-http-client:1.30.1'
+  compile 'com.google.http-client:google-http-client:1.34.2'
   compile "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   compile "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   compile "commons-logging:commons-logging:${versions.commonslogging}"
@@ -42,23 +42,23 @@ dependencies {
   compile 'com.google.api:api-common:1.8.1'
   compile 'com.google.api:gax:1.45.0'
   compile 'org.threeten:threetenbp:1.3.3'
-  compile 'com.google.protobuf:protobuf-java-util:3.7.1'
-  compile 'com.google.protobuf:protobuf-java:3.7.1'
+  compile 'com.google.protobuf:protobuf-java-util:3.11.3'
+  compile 'com.google.protobuf:protobuf-java:3.11.3'
   compile 'com.google.code.gson:gson:2.7'
   compile 'com.google.api.grpc:proto-google-common-protos:1.16.0'
   compile 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
   compile 'com.google.cloud:google-cloud-core-http:1.77.0'
-  compile 'com.google.auth:google-auth-library-credentials:0.16.1'
-  compile 'com.google.auth:google-auth-library-oauth2-http:0.16.1'
+  compile 'com.google.auth:google-auth-library-credentials:0.20.0'
+  compile 'com.google.auth:google-auth-library-oauth2-http:0.20.0'
   compile 'com.google.oauth-client:google-oauth-client:1.28.0'
-  compile 'com.google.api-client:google-api-client:1.28.0'
+  compile 'com.google.api-client:google-api-client:1.30.8'
   compile 'com.google.http-client:google-http-client-appengine:1.29.2'
   compile 'com.google.http-client:google-http-client-jackson2:1.29.2'
   compile 'com.google.api:gax-httpjson:0.62.0'
   compile 'io.grpc:grpc-context:1.12.0'
   compile 'io.opencensus:opencensus-api:0.18.0'
   compile 'io.opencensus:opencensus-contrib-http-util:0.18.0'
-  compile 'com.google.apis:google-api-services-storage:v1-rev20190426-1.28.0'
+  compile 'com.google.apis:google-api-services-storage:v1-rev20191011-1.30.3'
 
   // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
   // and whitelist this hack in JarHell
@@ -297,6 +297,9 @@ thirdPartyAudit.ignoreViolations(
   'com.google.protobuf.UnsafeUtil$1',
   'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
   'com.google.protobuf.UnsafeUtil$MemoryAccessor',
+  'com.google.protobuf.MessageSchema',
+  'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+  'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
   'com.google.common.cache.Striped64',
   'com.google.common.cache.Striped64$1',
   'com.google.common.cache.Striped64$Cell',

x-pack/snapshot-tool/licenses/google-api-client-1.30.8.jar.sha1

@@ -0,0 +1 @@
+b8664bfb9287f269228a4bbc36988174d4f311eb

x-pack/snapshot-tool/licenses/google-api-services-storage-v1-rev20191011-1.30.3.jar.sha1

@@ -0,0 +1 @@
+53fc173da380f9b65caad69912223db128dbf7ac

x-pack/snapshot-tool/licenses/google-auth-library-credentials-0.20.0.jar.sha1

@@ -0,0 +1 @@
+87a91a373e64ba5c3cdf8cc5cf54b189dd1492f8

x-pack/snapshot-tool/licenses/google-auth-library-oauth2-http-0.20.0.jar.sha1

@@ -0,0 +1 @@
+f33d4d6c91a68826816606a2208990eea93fcb2a

x-pack/snapshot-tool/licenses/google-cloud-core-1.92.5.jar.sha1

@@ -0,0 +1 @@
+97ca5c35f34ee67dde8f2d4e9b940f0a96393c66

x-pack/snapshot-tool/licenses/google-cloud-storage-1.104.0.jar.sha1

@@ -0,0 +1 @@
+8a083661f3732b21c0addd4fe56fd3f05a9c5a84

x-pack/snapshot-tool/licenses/google-http-client-1.34.2.jar.sha1

@@ -0,0 +1 @@
+58b4c500b5ce58271f1a6c5f6767aa8721158ea4

x-pack/snapshot-tool/licenses/protobuf-java-3.11.3.jar.sha1

@@ -0,0 +1 @@
+df12be70b968e32442821a2cfdc3cede5a42dec5

x-pack/snapshot-tool/licenses/protobuf-java-util-3.11.3.jar.sha1

@@ -0,0 +1 @@
+fd4ba2dfeb1b010eb20ca27e65fbfb74fbbdcdb9