Integrate from Piper for C++, Java, and Python

CHANGES.txt

@@ -20,6 +20,7 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
   * Fix repeated enum extension size in field listener
   * Enable Any Text Expansion for Descriptors::DebugString()
   * Switch from int{8,16,32,64} to int{8,16,32,64}_t
+  * Reduce memory usage of the DescriptorPool type.
 
   Java
   * Optimized FieldDescriptor.valueOf() to avoid array copying.
@@ -32,6 +33,9 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
   * Fix for optimization when reading doubles from binary wire format
   * Replace toArray implementation with toJSON.
 
+  Kotlin
+  * Suppress NOTHING_TO_INLINE in Kotlin generated inline functions.
+
 2021-06-04 version 3.17.3 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
   Python
   * Note: This is the last release to support Python 2.7. Future releases will

Makefile.am

@@ -1039,6 +1039,7 @@ python_EXTRA_DIST=                                                           \
   python/google/protobuf/internal/packed_field_test.proto                    \
   python/google/protobuf/internal/proto_builder_test.py                      \
   python/google/protobuf/internal/python_message.py                          \
+  python/google/protobuf/internal/python_protobuf.cc                         \
   python/google/protobuf/internal/reflection_test.py                         \
   python/google/protobuf/internal/service_reflection_test.py                 \
   python/google/protobuf/internal/symbol_database_test.py                    \
@@ -1093,6 +1094,7 @@ python_EXTRA_DIST=                                                           \
   python/google/protobuf/pyext/repeated_scalar_container.h                   \
   python/google/protobuf/pyext/safe_numerics.h                               \
   python/google/protobuf/pyext/scoped_pyobject_ptr.h                         \
+  python/google/protobuf/python_protobuf.h                                   \
   python/google/protobuf/reflection.py                                       \
   python/google/protobuf/service.py                                          \
   python/google/protobuf/service_reflection.py                               \

benchmarks/util/result_parser.py

@@ -300,29 +300,44 @@ def get_result_from_file(cpp_file="",
 
   return __results
 
+
 if __name__ == "__main__":
   parser = argparse.ArgumentParser()
-  parser.add_argument("-cpp", "--cpp_input_file",
-                      help="The CPP benchmark result file's name",
-                      default="")
-  parser.add_argument("-java", "--java_input_file",
-                      help="The Java benchmark result file's name",
-                      default="")
-  parser.add_argument("-python", "--python_input_file",
-                      help="The Python benchmark result file's name",
-                      default="")
-  parser.add_argument("-go", "--go_input_file",
-                      help="The golang benchmark result file's name",
-                      default="")
-  parser.add_argument("-node", "--node_input_file",
-                      help="The node.js benchmark result file's name",
-                      default="")
-  parser.add_argument("-php", "--php_input_file",
-                      help="The pure php benchmark result file's name",
-                      default="")
-  parser.add_argument("-php_c", "--php_c_input_file",
-                      help="The php with c ext benchmark result file's name",
-                      default="")
+  parser.add_argument(
+      "-cpp",
+      "--cpp_input_file",
+      help="The CPP benchmark result file's name",
+      default="")
+  parser.add_argument(
+      "-java",
+      "--java_input_file",
+      help="The Java benchmark result file's name",
+      default="")
+  parser.add_argument(
+      "-python",
+      "--python_input_file",
+      help="The Python benchmark result file's name",
+      default="")
+  parser.add_argument(
+      "-go",
+      "--go_input_file",
+      help="The golang benchmark result file's name",
+      default="")
+  parser.add_argument(
+      "-node",
+      "--node_input_file",
+      help="The node.js benchmark result file's name",
+      default="")
+  parser.add_argument(
+      "-php",
+      "--php_input_file",
+      help="The pure php benchmark result file's name",
+      default="")
+  parser.add_argument(
+      "-php_c",
+      "--php_c_input_file",
+      help="The php with c ext benchmark result file's name",
+      default="")
   args = parser.parse_args()
 
   results = get_result_from_file(

conformance/binary_json_conformance_suite.cc

@@ -219,21 +219,21 @@ string GetNonDefaultValue(FieldDescriptor::Type type) {
 #define UNKNOWN_FIELD 666
 
 enum class Packed {
-  UNSPECIFIED = 0,
-  TRUE = 1,
-  FALSE = 2,
+  kUnspecified = 0,
+  kTrue = 1,
+  kFalse = 2,
 };
 
 const FieldDescriptor* GetFieldForType(FieldDescriptor::Type type,
                                        bool repeated, bool is_proto3,
-                                       Packed packed = Packed::UNSPECIFIED) {
+                                       Packed packed = Packed::kUnspecified) {
   const Descriptor* d = is_proto3 ?
       TestAllTypesProto3().GetDescriptor() : TestAllTypesProto2().GetDescriptor();
   for (int i = 0; i < d->field_count(); i++) {
     const FieldDescriptor* f = d->field(i);
     if (f->type() == type && f->is_repeated() == repeated) {
-      if ((packed == Packed::TRUE && !f->is_packed()) ||
-          (packed == Packed::FALSE && f->is_packed())) {
+      if ((packed == Packed::kTrue && !f->is_packed()) ||
+          (packed == Packed::kFalse && f->is_packed())) {
         continue;
       }
       return f;
@@ -243,10 +243,10 @@ const FieldDescriptor* GetFieldForType(FieldDescriptor::Type type,
   string packed_string = "";
   const string repeated_string = repeated ? "Repeated " : "Singular ";
   const string proto_string = is_proto3 ? "Proto3" : "Proto2";
-  if (packed == Packed::TRUE) {
+  if (packed == Packed::kTrue) {
     packed_string = "Packed ";
   }
-  if (packed == Packed::FALSE) {
+  if (packed == Packed::kFalse) {
     packed_string = "Unpacked ";
   }
   GOOGLE_LOG(FATAL) << "Couldn't find field with type: " << repeated_string.c_str()
@@ -796,9 +796,9 @@ void BinaryAndJsonConformanceSuite::TestValidDataForType(
     // Test repeated fields.
     if (FieldDescriptor::IsTypePackable(type)) {
       const FieldDescriptor* packed_field =
-          GetFieldForType(type, true, is_proto3, Packed::TRUE);
+          GetFieldForType(type, true, is_proto3, Packed::kTrue);
       const FieldDescriptor* unpacked_field =
-          GetFieldForType(type, true, is_proto3, Packed::FALSE);
+          GetFieldForType(type, true, is_proto3, Packed::kFalse);
 
       string default_proto_packed;
       string default_proto_unpacked;

java/core/src/main/java/com/google/protobuf/CodedInputStream.java

@@ -87,7 +87,7 @@ public static CodedInputStream newInstance(final InputStream input, int bufferSi
       throw new IllegalArgumentException("bufferSize must be > 0");
     }
     if (input == null) {
-      // TODO(nathanmittler): Ideally we should throw here. This is done for backward compatibility.
+      // Ideally we would throw here. This is done for backward compatibility.
       return newInstance(EMPTY_BYTE_ARRAY);
     }
     return new StreamDecoder(input, bufferSize);
@@ -1146,7 +1146,7 @@ public int readRawLittleEndian32() throws IOException {
 
       final byte[] buffer = this.buffer;
       pos = tempPos + FIXED32_SIZE;
-      return (((buffer[tempPos] & 0xff))
+      return ((buffer[tempPos] & 0xff)
           | ((buffer[tempPos + 1] & 0xff) << 8)
           | ((buffer[tempPos + 2] & 0xff) << 16)
           | ((buffer[tempPos + 3] & 0xff) << 24));
@@ -1162,7 +1162,7 @@ public long readRawLittleEndian64() throws IOException {
 
       final byte[] buffer = this.buffer;
       pos = tempPos + FIXED64_SIZE;
-      return (((buffer[tempPos] & 0xffL))
+      return ((buffer[tempPos] & 0xffL)
           | ((buffer[tempPos + 1] & 0xffL) << 8)
           | ((buffer[tempPos + 2] & 0xffL) << 16)
           | ((buffer[tempPos + 3] & 0xffL) << 24)
@@ -1871,7 +1871,7 @@ public int readRawLittleEndian32() throws IOException {
       }
 
       pos = tempPos + FIXED32_SIZE;
-      return (((UnsafeUtil.getByte(tempPos) & 0xff))
+      return ((UnsafeUtil.getByte(tempPos) & 0xff)
           | ((UnsafeUtil.getByte(tempPos + 1) & 0xff) << 8)
           | ((UnsafeUtil.getByte(tempPos + 2) & 0xff) << 16)
           | ((UnsafeUtil.getByte(tempPos + 3) & 0xff) << 24));
@@ -1886,7 +1886,7 @@ public long readRawLittleEndian64() throws IOException {
       }
 
       pos = tempPos + FIXED64_SIZE;
-      return (((UnsafeUtil.getByte(tempPos) & 0xffL))
+      return ((UnsafeUtil.getByte(tempPos) & 0xffL)
           | ((UnsafeUtil.getByte(tempPos + 1) & 0xffL) << 8)
           | ((UnsafeUtil.getByte(tempPos + 2) & 0xffL) << 16)
           | ((UnsafeUtil.getByte(tempPos + 3) & 0xffL) << 24)
@@ -2014,11 +2014,16 @@ private ByteBuffer slice(long begin, long end) throws IOException {
       int prevPos = buffer.position();
       int prevLimit = buffer.limit();
       try {
+        // Casts to Buffer here are required for Java 8 compatibility
+        // no matter what tricorder tells you. see
+        // https://issues.apache.org/jira/browse/MRESOLVER-85
         ((Buffer) buffer).position(bufferPos(begin));
         ((Buffer) buffer).limit(bufferPos(end));
         return buffer.slice();
       } catch (IllegalArgumentException e) {
-        throw InvalidProtocolBufferException.truncatedMessage();
+        InvalidProtocolBufferException ex = InvalidProtocolBufferException.truncatedMessage();
+        ex.initCause(e);
+        throw ex;
       } finally {
         ((Buffer) buffer).position(prevPos);
         ((Buffer) buffer).limit(prevLimit);
@@ -2660,7 +2665,7 @@ public int readRawLittleEndian32() throws IOException {
 
       final byte[] buffer = this.buffer;
       pos = tempPos + FIXED32_SIZE;
-      return (((buffer[tempPos] & 0xff))
+      return ((buffer[tempPos] & 0xff)
           | ((buffer[tempPos + 1] & 0xff) << 8)
           | ((buffer[tempPos + 2] & 0xff) << 16)
           | ((buffer[tempPos + 3] & 0xff) << 24));
@@ -2987,7 +2992,7 @@ private List<byte[]> readRawBytesSlowPathRemainingChunks(int sizeLeft) throws IO
       // by allocating and reading only a small chunk at a time, so that the
       // malicious message must actually *be* extremely large to cause
       // problems.  Meanwhile, we limit the allowed size of a message elsewhere.
-      final List<byte[]> chunks = new ArrayList<byte[]>();
+      final List<byte[]> chunks = new ArrayList<>();
 
       while (sizeLeft > 0) {
         // TODO(nathanmittler): Consider using a value larger than DEFAULT_BUFFER_SIZE.
@@ -3134,16 +3139,16 @@ private void skipRawBytesSlowPath(final int size) throws IOException {
    */
   private static final class IterableDirectByteBufferDecoder extends CodedInputStream {
     /** The object that need to decode. */
-    private Iterable<ByteBuffer> input;
+    private final Iterable<ByteBuffer> input;
     /** The {@link Iterator} with type {@link ByteBuffer} of {@code input} */
-    private Iterator<ByteBuffer> iterator;
+    private final Iterator<ByteBuffer> iterator;
     /** The current ByteBuffer; */
     private ByteBuffer currentByteBuffer;
     /**
-     * If {@code true}, indicates that all the buffer are backing a {@link ByteString} and are
+     * If {@code true}, indicates that all the buffers are backing a {@link ByteString} and are
      * therefore considered to be an immutable input source.
      */
-    private boolean immutable;
+    private final boolean immutable;
     /**
      * If {@code true}, indicates that calls to read {@link ByteString} or {@code byte[]}
      * <strong>may</strong> return slices of the underlying buffer, rather than copies.
@@ -3516,8 +3521,7 @@ public ByteString readBytes() throws IOException {
           currentByteBufferPos += size;
           return result;
         } else {
-          byte[] bytes;
-          bytes = new byte[size];
+          byte[] bytes = new byte[size];
           UnsafeUtil.copyMemory(currentByteBufferPos, bytes, 0, size);
           currentByteBufferPos += size;
           return ByteString.wrap(bytes);
@@ -3738,7 +3742,7 @@ public int readRawLittleEndian32() throws IOException {
       if (currentRemaining() >= FIXED32_SIZE) {
         long tempPos = currentByteBufferPos;
         currentByteBufferPos += FIXED32_SIZE;
-        return (((UnsafeUtil.getByte(tempPos) & 0xff))
+        return ((UnsafeUtil.getByte(tempPos) & 0xff)
             | ((UnsafeUtil.getByte(tempPos + 1) & 0xff) << 8)
             | ((UnsafeUtil.getByte(tempPos + 2) & 0xff) << 16)
             | ((UnsafeUtil.getByte(tempPos + 3) & 0xff) << 24));
@@ -3754,7 +3758,7 @@ public long readRawLittleEndian64() throws IOException {
       if (currentRemaining() >= FIXED64_SIZE) {
         long tempPos = currentByteBufferPos;
         currentByteBufferPos += FIXED64_SIZE;
-        return (((UnsafeUtil.getByte(tempPos) & 0xffL))
+        return ((UnsafeUtil.getByte(tempPos) & 0xffL)
             | ((UnsafeUtil.getByte(tempPos + 1) & 0xffL) << 8)
             | ((UnsafeUtil.getByte(tempPos + 2) & 0xffL) << 16)
             | ((UnsafeUtil.getByte(tempPos + 3) & 0xffL) << 24)
@@ -3875,11 +3879,6 @@ public byte[] readRawBytes(final int length) throws IOException {
      * Try to get raw bytes from {@code input} with the size of {@code length} and copy to {@code
      * bytes} array. If the size is bigger than the number of remaining bytes in the input, then
      * throw {@code truncatedMessage} exception.
-     *
-     * @param bytes
-     * @param offset
-     * @param length
-     * @throws IOException
      */
     private void readRawBytesTo(byte[] bytes, int offset, final int length) throws IOException {
       if (length >= 0 && length <= remaining()) {
@@ -3966,6 +3965,9 @@ private ByteBuffer slice(int begin, int end) throws IOException {
       int prevPos = currentByteBuffer.position();
       int prevLimit = currentByteBuffer.limit();
       try {
+        // casts to Buffer here are required for Java 8 compatibility
+        // no matter what tricorder tells you. see
+        // https://issues.apache.org/jira/browse/MRESOLVER-85
         ((Buffer) currentByteBuffer).position(begin);
         ((Buffer) currentByteBuffer).limit(end);
         return currentByteBuffer.slice();

java/core/src/test/java/com/google/protobuf/ExtensionRegistryFactoryTest.java

@@ -57,7 +57,7 @@
  * <p>The test mechanism employed here is based on the pattern in {@code
  * com.google.common.util.concurrent.AbstractFutureFallbackAtomicHelperTest}
  *
- * <p> This test is temporarily disabled while we figure out how to fix the class loading used for
+ * <p>This test is temporarily disabled while we figure out how to fix the class loading used for
  * testing lite functionality.
  */
 @SuppressWarnings("JUnit4ClassUsedInJUnit3")

java/core/src/test/java/com/google/protobuf/TestUtil.java

@@ -3758,7 +3758,9 @@ public void assertReflectionRepeatedSettersRejectNull(Message.Builder builder)
 
   /** @param filePath The path relative to {@link #getTestDataDir}. */
   public static String readTextFromFile(String filePath) {
-    return readBytesFromFile(filePath).toStringUtf8().replace(System.getProperty("line.separator"), "\n");
+    return readBytesFromFile(filePath)
+        .toStringUtf8()
+        .replace(System.getProperty("line.separator"), "\n");
   }
 
   private static File getTestDataDir() {

java/kotlin/src/main/kotlin/com/google/protobuf/ExtensionList.kt

@@ -34,14 +34,14 @@ import com.google.protobuf.ExtensionLite
 import com.google.protobuf.MessageLite
 
 /**
- * Implementation for ExtensionList and ExtensionListLite.  Like [DslList], represents an
- * unmodifiable view of a repeated proto field -- in this case, an extension field -- but
- * supports querying the extension.
+ * Implementation for ExtensionList and ExtensionListLite. Like [DslList], represents an
+ * unmodifiable view of a repeated proto field -- in this case, an extension field -- but supports
+ * querying the extension.
  */
-class ExtensionList<E, M : MessageLite> @OnlyForUseByGeneratedProtoCode constructor(
-  val extension: ExtensionLite<M, List<E>>,
-  private val delegate: List<E>
-) : List<E> by delegate {
+class ExtensionList<E, M : MessageLite>
+@OnlyForUseByGeneratedProtoCode
+constructor(val extension: ExtensionLite<M, List<E>>, private val delegate: List<E>) :
+  List<E> by delegate {
   override fun iterator(): Iterator<E> = UnmodifiableIterator(delegate.iterator())
 
   override fun listIterator(): ListIterator<E> = UnmodifiableListIterator(delegate.listIterator())