From f466b4901dee2f20c83779929952ca3638005482 Mon Sep 17 00:00:00 2001 From: Les Vogel Date: Thu, 16 Feb 2017 21:55:26 -0800 Subject: [PATCH] Get full IT working again (#523) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Spanner Works 1. remove comments from pom.xml 2. explicitly pass the SystemPropertyValues in local pom — not sure how it worked earlier. 3. Fix a comment in the QuickStart sample 4. Restore stdOut in both IT samples * Minor README tweak storage-transfer * ErrorProne 1. Turn on ErrorProne - `mvn clean verify -DskipTests` passes 2. Fixes for ErrorProne - mostly removing compiler plugin, java 1.8 for most samples. (No more Java 5) 3. Contributing now mentions gradle and testing 4. Fix storage-transfer README 5. back out some changes to MAVEN_OPTS * Debugging Magic 1. What project are these tests being run on? 2. Skip spanner for now. * change project_ID 1. enabled cloud-samples-tests 2. change the project 3. remove the maven memory hack. 4. remove the debugging code. * revert to j-d-s-t for projectID that BQ uses --- .travis.yml | 2 +- CONTRIBUTING.md | 39 ++++++++++++++- datastore/pom.xml | 16 +++---- logging/pom.xml | 20 ++------ monitoring/v2/pom.xml | 20 ++------ pom.xml | 11 +---- spanner/cloud-client/pom.xml | 12 +++++ .../com/example/spanner/QuickstartSample.java | 2 +- .../example/spanner/QuickstartSampleIT.java | 3 +- .../com/example/spanner/SpannerSampleIT.java | 6 ++- speech/grpc/pom.xml | 48 ++----------------- .../speech/StreamingRecognizeClient.java | 2 +- storage/storage-transfer/README.md | 25 +++++----- travis.sh | 2 +- 14 files changed, 96 insertions(+), 112 deletions(-) diff --git a/.travis.yml b/.travis.yml index f9dbf428c84..bdf2e33a92e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,8 +42,8 @@ before_install: - openssl aes-256-cbc -K $encrypted_eb858daba67b_key -iv $encrypted_eb858daba67b_iv -in secrets.env.enc -out secrets.env -d && set +x && source secrets.env && set -x || true +# Cross project - GOOGLE_APPLICATION_CREDENTIALS uses cloud-docs-tests, but BQ uses G_C_P (argh!) - export GOOGLE_CLOUD_PROJECT=java-docs-samples-tests -- export MAVEN_OPTS='-Xmx6g -Xms1g -XX:-UseGCOverheadLimit -XX:+UseG1GC -XX:+PrintFlagsInitial -XX:+PrintFlagsFinal' # Skip the install step, since Maven will download the dependencies we need # when the test build runs. # http://stackoverflow.com/q/31945809/101923 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e992c3e68b..11292584458 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,11 @@ # How to become a contributor and submit your own code +* [Contributor License Agreements](#Contributor-License-Agreements) +* [Contributing a Patch](#Contributing-a-Patch) +* [Build Tools](#build-tools) +* [Integration Testing](#testing) +* [Style](#Style) + ## Contributor License Agreements We'd love to accept your sample apps and patches! Before we can take them, we @@ -31,6 +37,35 @@ accept your pull requests. 1. Ensure that your code has an appropriate set of unit tests which all pass. 1. Submit a pull request. +## Build Tools + +All new samples should build and run integration tests with both [Maven](https://maven.apache.org/) and [Gradle](https://gradle.org/). + +## Testing + +All samples must have Integration Tests (ie. They need to run against a real service) that run with +`mvn verify` & `gradle build test`. If we need to enable an API, let us know. + +Your `build.gradle` should have the following section: + +```groovy + +test { + useJUnit() + testLogging.showStandardStreams = true + beforeTest { descriptor -> + logger.lifecycle("test: " + descriptor + " Running") + } + + onOutput { descriptor, event -> + logger.lifecycle("test: " + descriptor + ": " + event.message ) + } + afterTest { descriptor, result -> + logger.lifecycle("test: " + descriptor + ": " + result ) + } +} +``` + ## Style Samples in this repository follow the [Google Java Style Guide][java-style]. @@ -47,7 +82,7 @@ tool or IntelliJ plugin. ### Running the Linter -To run the checkstyle plugin on an existing sample, run +To run the checkstyle & ErrorProne plugins on an existing sample, run ```shell mvn clean verify -DskipTests @@ -73,3 +108,5 @@ uses the common Checkstyle configuration. ``` This is just used for testing. The sample should build without a parent defined. + + diff --git a/datastore/pom.xml b/datastore/pom.xml index f183a4e1908..d3e20768975 100644 --- a/datastore/pom.xml +++ b/datastore/pom.xml @@ -31,6 +31,12 @@ Example snippets for Datastore concepts and getting started documentation. + + + 1.8 + 1.8 + + com.google.cloud @@ -49,15 +55,7 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 2.5.1 - - 1.7 - 1.7 - - + diff --git a/logging/pom.xml b/logging/pom.xml index 082ae74049d..2113c23ebc3 100644 --- a/logging/pom.xml +++ b/logging/pom.xml @@ -12,6 +12,11 @@ .. + + 1.8 + 1.8 + + com.google.apis @@ -43,19 +48,4 @@ - - src/main/java - - - org.apache.maven.plugins - maven-compiler-plugin - 3.2 - - 5 - 5 - - - - - diff --git a/monitoring/v2/pom.xml b/monitoring/v2/pom.xml index e67af8248ce..2c2b3d9d6a4 100644 --- a/monitoring/v2/pom.xml +++ b/monitoring/v2/pom.xml @@ -20,6 +20,11 @@ + + 1.8 + 1.8 + + com.google.apis @@ -68,19 +73,4 @@ - - src/main/java - - - org.apache.maven.plugins - maven-compiler-plugin - 3.2 - - 5 - 5 - - - - - diff --git a/pom.xml b/pom.xml index d5401ec24b2..c49ff8fa61b 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ com.google.cloud.samples shared-configuration - 1.0.1 + 1.0.2 @@ -142,14 +142,7 @@ appengine-api-1.0-sdk ${appengine.sdk.version} - + jstl jstl diff --git a/spanner/cloud-client/pom.xml b/spanner/cloud-client/pom.xml index 990954738e9..b0b00bc4a5b 100644 --- a/spanner/cloud-client/pom.xml +++ b/spanner/cloud-client/pom.xml @@ -95,6 +95,18 @@ + + org.apache.maven.plugins + maven-failsafe-plugin + 2.19.1 + + + ${spanner.test.instance} + ${spanner.sample.database} + ${spanner.quickstart.database} + + + diff --git a/spanner/cloud-client/src/main/java/com/example/spanner/QuickstartSample.java b/spanner/cloud-client/src/main/java/com/example/spanner/QuickstartSample.java index eeed92ff811..6b0fdc69c8c 100644 --- a/spanner/cloud-client/src/main/java/com/example/spanner/QuickstartSample.java +++ b/spanner/cloud-client/src/main/java/com/example/spanner/QuickstartSample.java @@ -40,7 +40,7 @@ public static void main(String... args) throws Exception { SpannerOptions options = SpannerOptions.newBuilder().build(); Spanner spanner = options.getService(); - // Name of your database. Eg: projects/my-project/instances/instanceId/databases/databaseId + // Name of your instance & database. String instanceId = args[0]; String databaseId = args[1]; try { diff --git a/spanner/cloud-client/src/test/java/com/example/spanner/QuickstartSampleIT.java b/spanner/cloud-client/src/test/java/com/example/spanner/QuickstartSampleIT.java index 007fa464f1b..15df8e810fe 100644 --- a/spanner/cloud-client/src/test/java/com/example/spanner/QuickstartSampleIT.java +++ b/spanner/cloud-client/src/test/java/com/example/spanner/QuickstartSampleIT.java @@ -37,6 +37,7 @@ public class QuickstartSampleIT { // This database needs to exist for test to pass. private String dbId = System.getProperty("spanner.quickstart.database"); private ByteArrayOutputStream bout; + private PrintStream stdOut = System.out; private PrintStream out; @Before @@ -48,7 +49,7 @@ public void setUp() { @After public void tearDown() { - System.setOut(null); + System.setOut(stdOut); } @Test diff --git a/spanner/cloud-client/src/test/java/com/example/spanner/SpannerSampleIT.java b/spanner/cloud-client/src/test/java/com/example/spanner/SpannerSampleIT.java index 64eb8208071..fa1b9939a4d 100644 --- a/spanner/cloud-client/src/test/java/com/example/spanner/SpannerSampleIT.java +++ b/spanner/cloud-client/src/test/java/com/example/spanner/SpannerSampleIT.java @@ -39,16 +39,18 @@ @SuppressWarnings("checkstyle:abbreviationaswordinname") public class SpannerSampleIT { // The instance needs to exist for tests to pass. - String instanceId = System.getProperty("spanner.test.instance"); - String databaseId = System.getProperty("spanner.sample.database"); + private final String instanceId = System.getProperty("spanner.test.instance"); + private final String databaseId = System.getProperty("spanner.sample.database"); DatabaseId dbId; DatabaseAdminClient dbClient; private String runSample(String command) throws Exception { + PrintStream stdOut = System.out; ByteArrayOutputStream bout = new ByteArrayOutputStream(); PrintStream out = new PrintStream(bout); System.setOut(out); SpannerSample.main(new String[]{command, instanceId, databaseId}); + System.setOut(stdOut); return bout.toString(); } diff --git a/speech/grpc/pom.xml b/speech/grpc/pom.xml index 08349ef10f1..2030c90a54f 100644 --- a/speech/grpc/pom.xml +++ b/speech/grpc/pom.xml @@ -52,28 +52,10 @@ limitations under the License. 1.0.3 3.6.0 0.5.0 - - - - jdk7 - - 1.7 - - - 1.7 - - - - jdk8 - - 1.8 - - - 1.8 - - - + 1.8 + 1.8 + @@ -156,34 +138,10 @@ limitations under the License. - - - - never - - - false - - central - Central Repository - https://repo.maven.apache.org/maven2 - - - - maven-compiler-plugin - ${maven-compiler-plugin-version} - - ${jdk.version} - ${jdk.version} - true - false - -Xlint:-options - - maven-assembly-plugin diff --git a/speech/grpc/src/main/java/com/examples/cloud/speech/StreamingRecognizeClient.java b/speech/grpc/src/main/java/com/examples/cloud/speech/StreamingRecognizeClient.java index e7f17e1d29a..7a70b0359fb 100644 --- a/speech/grpc/src/main/java/com/examples/cloud/speech/StreamingRecognizeClient.java +++ b/speech/grpc/src/main/java/com/examples/cloud/speech/StreamingRecognizeClient.java @@ -124,7 +124,7 @@ private TargetDataLine getAudioInputLine() { DataLine.Info info = new DataLine.Info(TargetDataLine.class, format); if (!AudioSystem.isLineSupported(info)) { throw new RuntimeException(String.format( - "Device doesn't support LINEAR16 mono raw audio format at {}Hz", samplingRate)); + "Device doesn't support LINEAR16 mono raw audio format at {%d}Hz", samplingRate)); } try { TargetDataLine line = (TargetDataLine) AudioSystem.getLine(info); diff --git a/storage/storage-transfer/README.md b/storage/storage-transfer/README.md index 6c7fb8dd862..5fcb1830f9f 100644 --- a/storage/storage-transfer/README.md +++ b/storage/storage-transfer/README.md @@ -54,18 +54,21 @@ Creating a one-time transfer from Amazon S3 to Google Cloud Storage. 1. Set the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables. 1. In AwsRequester.java, fill in the user-provided constants. 1. Compile the package with - ``` - mvn compile - ``` + +```bash +mvn compile +``` + 1. Run the transfer job with - ``` - mvn exec:java \ - -Dexec.mainClass="com.google.cloud.storage.storagetransfer.samples.AwsRequester" \ - -DprojectId=your-google-cloud-project-id \ - -DjobDescription="Sample transfer job from S3 to GCS." \ - -DawsSourceBucket=your-s3-bucket-name \ - -DgcsSinkBucket=your-gcs-bucket-name - ``` + +```bash +mvn exec:java \ + -Dexec.mainClass="com.google.cloud.storage.storagetransfer.samples.AwsRequester" \ + -DprojectId=your-google-cloud-project-id \ + -DjobDescription="Sample transfer job from S3 to GCS." \ + -DawsSourceBucket=your-s3-bucket-name \ + -DgcsSinkBucket=your-gcs-bucket-name +``` 1. Note the job ID in the returned Transfer Job. ## Transfer data from a standard Cloud Storage bucket to a Cloud Storage Nearline bucket diff --git a/travis.sh b/travis.sh index 08e87583510..60896e4d390 100755 --- a/travis.sh +++ b/travis.sh @@ -64,7 +64,7 @@ common_travis_dir="$(travis_changed_files_parent)" [ -z "$common_travis_dir" ] || pushd "$common_travis_dir" -./mvnw --batch-mode clean verify -DskipTests=$SKIP_TESTS | egrep -v "(^\[INFO\] Download|^\[INFO\].*skipping)" +./mvnw --batch-mode clean verify -e -DskipTests=$SKIP_TESTS | egrep -v "(^\[INFO\] Download|^\[INFO\].*skipping)" [ -z "$common_travis_dir" ] || popd