Integrated internal changes from Google

Makefile.am

@@ -605,6 +605,8 @@ php_EXTRA_DIST=                                              \
   php/tests/test.proto                                       \
   php/tests/test.pb.php                                      \
   php/tests/memory_leak_test.php                             \
+  php/tests/google/protobuf/empty.pb.php                     \
+  php/tests/well_known_test.php                              \
   php/README.md                                              \
   php/ext/google/protobuf/utf8.h                             \
   php/ext/google/protobuf/message.c                          \
@@ -712,6 +714,7 @@ python_EXTRA_DIST=                                                           \
   python/google/protobuf/pyext/repeated_composite_container.h                \
   python/google/protobuf/pyext/repeated_scalar_container.cc                  \
   python/google/protobuf/pyext/repeated_scalar_container.h                   \
+  python/google/protobuf/pyext/safe_numerics.h                               \
   python/google/protobuf/pyext/scoped_pyobject_ptr.h                         \
   python/google/protobuf/reflection.py                                       \
   python/google/protobuf/service.py                                          \

cmake/libprotoc.cmake

@@ -72,6 +72,7 @@ set(libprotoc_files
   ${protobuf_source_dir}/src/google/protobuf/compiler/javanano/javanano_message_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/javanano/javanano_primitive_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/js/js_generator.cc
+  ${protobuf_source_dir}/src/google/protobuf/compiler/js/well_known_types_embed.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_enum_field.cc
   ${protobuf_source_dir}/src/google/protobuf/compiler/objectivec/objectivec_extension.cc

conformance/ConformanceJava.java

@@ -1,8 +1,11 @@
-
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.conformance.Conformance;
-import com.google.protobuf.util.JsonFormat;
 import com.google.protobuf.util.JsonFormat.TypeRegistry;
-import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.util.JsonFormat;
+import java.io.IOException;
+import java.nio.ByteBuffer;
 
 class ConformanceJava {
   private int testCount = 0;
@@ -47,13 +50,182 @@ private void writeLittleEndianIntToStdout(int val) throws Exception {
     writeToStdout(buf);
   }
 
+  private enum BinaryDecoder {
+    BYTE_STRING_DECODER() {
+      @Override
+      public Conformance.TestAllTypes parse(ByteString bytes)
+          throws InvalidProtocolBufferException {
+        return Conformance.TestAllTypes.parseFrom(bytes);
+      }
+    },
+    BYTE_ARRAY_DECODER() {
+      @Override
+      public Conformance.TestAllTypes parse(ByteString bytes)
+          throws InvalidProtocolBufferException {
+        return Conformance.TestAllTypes.parseFrom(bytes.toByteArray());
+      }
+    },
+    ARRAY_BYTE_BUFFER_DECODER() {
+      @Override
+      public Conformance.TestAllTypes parse(ByteString bytes)
+          throws InvalidProtocolBufferException {
+        ByteBuffer buffer = ByteBuffer.allocate(bytes.size());
+        bytes.copyTo(buffer);
+        buffer.flip();
+        try {
+          return Conformance.TestAllTypes.parseFrom(CodedInputStream.newInstance(buffer));
+        } catch (InvalidProtocolBufferException e) {
+          throw e;
+        } catch (IOException e) {
+          throw new RuntimeException(
+              "ByteString based ByteBuffer should not throw IOException.", e);
+        }
+      }
+    },
+    READONLY_ARRAY_BYTE_BUFFER_DECODER() {
+      @Override
+      public Conformance.TestAllTypes parse(ByteString bytes)
+          throws InvalidProtocolBufferException {
+        try {
+          return Conformance.TestAllTypes.parseFrom(
+              CodedInputStream.newInstance(bytes.asReadOnlyByteBuffer()));
+        } catch (InvalidProtocolBufferException e) {
+          throw e;
+        } catch (IOException e) {
+          throw new RuntimeException(
+              "ByteString based ByteBuffer should not throw IOException.", e);
+        }
+      }
+    },
+    DIRECT_BYTE_BUFFER_DECODER() {
+      @Override
+      public Conformance.TestAllTypes parse(ByteString bytes)
+          throws InvalidProtocolBufferException {
+        ByteBuffer buffer = ByteBuffer.allocateDirect(bytes.size());
+        bytes.copyTo(buffer);
+        buffer.flip();
+        try {
+          return Conformance.TestAllTypes.parseFrom(CodedInputStream.newInstance(buffer));
+        } catch (InvalidProtocolBufferException e) {
+          throw e;
+        } catch (IOException e) {
+          throw new RuntimeException(
+              "ByteString based ByteBuffer should not throw IOException.", e);
+        }
+      }
+    },
+    READONLY_DIRECT_BYTE_BUFFER_DECODER() {
+      @Override
+      public Conformance.TestAllTypes parse(ByteString bytes)
+          throws InvalidProtocolBufferException {
+        ByteBuffer buffer = ByteBuffer.allocateDirect(bytes.size());
+        bytes.copyTo(buffer);
+        buffer.flip();
+        try {
+          return Conformance.TestAllTypes.parseFrom(
+              CodedInputStream.newInstance(buffer.asReadOnlyBuffer()));
+        } catch (InvalidProtocolBufferException e) {
+          throw e;
+        } catch (IOException e) {
+          throw new RuntimeException(
+              "ByteString based ByteBuffer should not throw IOException.", e);
+        }
+      }
+    },
+    INPUT_STREAM_DECODER() {
+      @Override
+      public Conformance.TestAllTypes parse(ByteString bytes)
+          throws InvalidProtocolBufferException {
+        try {
+          return Conformance.TestAllTypes.parseFrom(bytes.newInput());
+        } catch (InvalidProtocolBufferException e) {
+          throw e;
+        } catch (IOException e) {
+          throw new RuntimeException(
+              "ByteString based InputStream should not throw IOException.", e);
+        }
+      }
+    };
+
+    public abstract Conformance.TestAllTypes parse(ByteString bytes)
+        throws InvalidProtocolBufferException;
+  }
+
+  private Conformance.TestAllTypes parseBinary(ByteString bytes)
+      throws InvalidProtocolBufferException {
+    Conformance.TestAllTypes[] messages =
+        new Conformance.TestAllTypes[BinaryDecoder.values().length];
+    InvalidProtocolBufferException[] exceptions =
+        new InvalidProtocolBufferException[BinaryDecoder.values().length];
+
+    boolean hasMessage = false;
+    boolean hasException = false;
+    for (int i = 0; i < BinaryDecoder.values().length; ++i) {
+      try {
+        messages[i] = BinaryDecoder.values()[i].parse(bytes);
+        hasMessage = true;
+      } catch (InvalidProtocolBufferException e) {
+        exceptions[i] = e;
+        hasException = true;
+      }
+    }
+
+    if (hasMessage && hasException) {
+      StringBuilder sb =
+          new StringBuilder("Binary decoders disagreed on whether the payload was valid.\n");
+      for (int i = 0; i < BinaryDecoder.values().length; ++i) {
+        sb.append(BinaryDecoder.values()[i].name());
+        if (messages[i] != null) {
+          sb.append(" accepted the payload.\n");
+        } else {
+          sb.append(" rejected the payload.\n");
+        }
+      }
+      throw new RuntimeException(sb.toString());
+    }
+
+    if (hasException) {
+      // We do not check if exceptions are equal. Different implementations may return different
+      // exception messages. Throw an arbitrary one out instead.
+      throw exceptions[0];
+    }
+
+    // Fast path comparing all the messages with the first message, assuming equality being
+    // symmetric and transitive.
+    boolean allEqual = true;
+    for (int i = 1; i < messages.length; ++i) {
+      if (!messages[0].equals(messages[i])) {
+        allEqual = false;
+        break;
+      }
+    }
+
+    // Slow path: compare and find out all unequal pairs.
+    if (!allEqual) {
+      StringBuilder sb = new StringBuilder();
+      for (int i = 0; i < messages.length - 1; ++i) {
+        for (int j = i + 1; j < messages.length; ++j) {
+          if (!messages[i].equals(messages[j])) {
+            sb.append(BinaryDecoder.values()[i].name())
+                .append(" and ")
+                .append(BinaryDecoder.values()[j].name())
+                .append(" parsed the payload differently.\n");
+          }
+        }
+      }
+      throw new RuntimeException(sb.toString());
+    }
+
+    return messages[0];
+  }
+
   private Conformance.ConformanceResponse doTest(Conformance.ConformanceRequest request) {
     Conformance.TestAllTypes testMessage;
 
     switch (request.getPayloadCase()) {
       case PROTOBUF_PAYLOAD: {
         try {
-          testMessage = Conformance.TestAllTypes.parseFrom(request.getProtobufPayload());
+          testMessage = parseBinary(request.getProtobufPayload());
         } catch (InvalidProtocolBufferException e) {
           return Conformance.ConformanceResponse.newBuilder().setParseError(e.getMessage()).build();
         }

conformance/conformance_test.cc

@@ -685,7 +685,7 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
       R"({
         "fieldname1": 1,
         "fieldName2": 2,
-        "fieldName3": 3,
+        "FieldName3": 3,
         "fieldName4": 4
       })",
       R"(
@@ -725,12 +725,12 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
   RunValidJsonTest(
       "FieldNameWithDoubleUnderscores", RECOMMENDED,
       R"({
-        "fieldName13": 13,
-        "fieldName14": 14,
+        "FieldName13": 13,
+        "FieldName14": 14,
         "fieldName15": 15,
         "fieldName16": 16,
         "fieldName17": 17,
-        "fieldName18": 18
+        "FieldName18": 18
       })",
       R"(
         __field_name13: 13
@@ -873,21 +873,19 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
         "optionalNestedMessage": {a: 1},
         "optional_nested_message": {}
       })");
-  // NOTE: The spec for JSON support is still being sorted out, these may not
-  // all be correct.
   // Serializers should use lowerCamelCase by default.
   RunValidJsonTestWithValidator(
       "FieldNameInLowerCamelCase", REQUIRED,
       R"({
         "fieldname1": 1,
         "fieldName2": 2,
-        "fieldName3": 3,
+        "FieldName3": 3,
         "fieldName4": 4
       })",
       [](const Json::Value& value) {
         return value.isMember("fieldname1") &&
             value.isMember("fieldName2") &&
-            value.isMember("fieldName3") &&
+            value.isMember("FieldName3") &&
             value.isMember("fieldName4");
       });
   RunValidJsonTestWithValidator(
@@ -921,20 +919,20 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
   RunValidJsonTestWithValidator(
       "FieldNameWithDoubleUnderscores", RECOMMENDED,
       R"({
-        "fieldName13": 13,
-        "fieldName14": 14,
+        "FieldName13": 13,
+        "FieldName14": 14,
         "fieldName15": 15,
         "fieldName16": 16,
         "fieldName17": 17,
-        "fieldName18": 18
+        "FieldName18": 18
       })",
       [](const Json::Value& value) {
-        return value.isMember("fieldName13") &&
-            value.isMember("fieldName14") &&
+        return value.isMember("FieldName13") &&
+            value.isMember("FieldName14") &&
             value.isMember("fieldName15") &&
             value.isMember("fieldName16") &&
             value.isMember("fieldName17") &&
-            value.isMember("fieldName18");
+            value.isMember("FieldName18");
       });
 
   // Integer fields.

conformance/failure_list_cpp.txt

@@ -17,9 +17,6 @@ Recommended.JsonInput.FieldNameDuplicate
 Recommended.JsonInput.FieldNameDuplicateDifferentCasing1
 Recommended.JsonInput.FieldNameDuplicateDifferentCasing2
 Recommended.JsonInput.FieldNameNotQuoted
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.JsonOutput
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.ProtobufOutput
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.Validator
 Recommended.JsonInput.MapFieldValueIsNull
 Recommended.JsonInput.RepeatedFieldMessageElementIsNull
 Recommended.JsonInput.RepeatedFieldPrimitiveElementIsNull
@@ -35,10 +32,6 @@ Recommended.JsonInput.TrailingCommaInAnObject
 Recommended.JsonInput.TrailingCommaInAnObjectWithNewlines
 Recommended.JsonInput.TrailingCommaInAnObjectWithSpace
 Recommended.JsonInput.TrailingCommaInAnObjectWithSpaceCommaSpace
-Required.JsonInput.DoubleFieldTooSmall
-Required.JsonInput.FieldNameInLowerCamelCase.Validator
-Required.JsonInput.FieldNameInSnakeCase.JsonOutput
-Required.JsonInput.FieldNameInSnakeCase.ProtobufOutput
 Required.ProtobufInput.PrematureEofBeforeKnownRepeatedValue.MESSAGE
 Required.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
 Required.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE

conformance/failure_list_csharp.txt

@@ -1,6 +0,0 @@
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.JsonOutput
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.ProtobufOutput
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.Validator
-Required.JsonInput.FieldNameInLowerCamelCase.Validator
-Required.JsonInput.FieldNameInSnakeCase.JsonOutput
-Required.JsonInput.FieldNameInSnakeCase.ProtobufOutput

conformance/failure_list_java.txt

@@ -20,9 +20,6 @@ Recommended.JsonInput.DoubleFieldNegativeInfinityNotQuoted
 Recommended.JsonInput.FieldMaskInvalidCharacter
 Recommended.JsonInput.FieldNameDuplicate
 Recommended.JsonInput.FieldNameNotQuoted
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.JsonOutput
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.ProtobufOutput
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.Validator
 Recommended.JsonInput.FloatFieldInfinityNotQuoted
 Recommended.JsonInput.FloatFieldNanNotQuoted
 Recommended.JsonInput.FloatFieldNegativeInfinityNotQuoted
@@ -38,12 +35,11 @@ Recommended.JsonInput.StringFieldUnpairedLowSurrogate
 Recommended.JsonInput.Uint32MapFieldKeyNotQuoted
 Recommended.JsonInput.Uint64MapFieldKeyNotQuoted
 Required.JsonInput.EnumFieldNotQuoted
-Required.JsonInput.FieldNameInLowerCamelCase.Validator
-Required.JsonInput.FieldNameInSnakeCase.JsonOutput
-Required.JsonInput.FieldNameInSnakeCase.ProtobufOutput
 Required.JsonInput.Int32FieldLeadingZero
 Required.JsonInput.Int32FieldNegativeWithLeadingZero
 Required.JsonInput.Int32FieldPlusSign
 Required.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool
 Required.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt
 Required.JsonInput.StringFieldNotAString
+Required.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
+Required.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE

conformance/failure_list_python.txt

@@ -1,18 +1,12 @@
 Recommended.JsonInput.DoubleFieldInfinityNotQuoted
 Recommended.JsonInput.DoubleFieldNanNotQuoted
 Recommended.JsonInput.DoubleFieldNegativeInfinityNotQuoted
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.JsonOutput
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.ProtobufOutput
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.Validator
 Recommended.JsonInput.FloatFieldInfinityNotQuoted
 Recommended.JsonInput.FloatFieldNanNotQuoted
 Recommended.JsonInput.FloatFieldNegativeInfinityNotQuoted
 Required.JsonInput.BytesFieldInvalidBase64Characters
 Required.JsonInput.DoubleFieldTooSmall
 Required.JsonInput.EnumFieldUnknownValue.Validator
-Required.JsonInput.FieldNameInLowerCamelCase.Validator
-Required.JsonInput.FieldNameInSnakeCase.JsonOutput
-Required.JsonInput.FieldNameInSnakeCase.ProtobufOutput
 Required.JsonInput.FloatFieldTooLarge
 Required.JsonInput.FloatFieldTooSmall
 Required.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool

conformance/failure_list_python_cpp.txt

@@ -10,18 +10,12 @@
 Recommended.JsonInput.DoubleFieldInfinityNotQuoted
 Recommended.JsonInput.DoubleFieldNanNotQuoted
 Recommended.JsonInput.DoubleFieldNegativeInfinityNotQuoted
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.JsonOutput
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.ProtobufOutput
-Recommended.JsonInput.FieldNameWithDoubleUnderscores.Validator
 Recommended.JsonInput.FloatFieldInfinityNotQuoted
 Recommended.JsonInput.FloatFieldNanNotQuoted
 Recommended.JsonInput.FloatFieldNegativeInfinityNotQuoted
 Required.JsonInput.BytesFieldInvalidBase64Characters
 Required.JsonInput.DoubleFieldTooSmall
 Required.JsonInput.EnumFieldUnknownValue.Validator
-Required.JsonInput.FieldNameInLowerCamelCase.Validator
-Required.JsonInput.FieldNameInSnakeCase.JsonOutput
-Required.JsonInput.FieldNameInSnakeCase.ProtobufOutput
 Required.JsonInput.FloatFieldTooLarge
 Required.JsonInput.FloatFieldTooSmall
 Required.JsonInput.RepeatedFieldWrongElementTypeExpectingIntegersGotBool