generate definitions / kversion: partial KIP-516 support

Some of KIP-516 was added to Kafka; this ports what was added:
- metadata
- leader and ISR
- update metadata

generate/definitions/03_metadata

@@ -1,11 +1,16 @@
 // MetadataRequest requests metadata from Kafka.
-MetadataRequest => key 3, max version 9, flexible v9+
+MetadataRequest => key 3, max version 10, flexible v9+
   // Topics is a list of topics to return metadata about. If this is null
   // in v1+, all topics are included. If this is empty, no topics are.
   // For v0 (<Kafka 0.10.0.0), if this is empty, all topics are included.
   Topics: nullable-v1+[=>]
-    // Topic is the topic to request metadata for.
-    Topic: string
+    // The topic ID. Only one of either topic ID or topic name should be used.
+    // If using the topic name, this should just be the default empty value.
+    TopicID: uuid // v10+
+    // Topic is the topic to request metadata for. Version 10 switched this
+    // from a string to a nullable string; if using a topic ID, this field
+    // should be null.
+    Topic: nullable-string-v10+
   // AllowAutoTopicCreation, introduced in Kafka 0.11.0.0, allows topic
   // auto creation of the topics in this request if they do not exist.
   AllowAutoTopicCreation: bool // v4+
@@ -59,6 +64,8 @@ MetadataResponse =>
     ErrorCode: int16
     // Topic is the topic this metadata corresponds to.
     Topic: string
+    // The topic ID.
+    TopicID: uuid // v10+
     // IsInternal signifies whether this topic is a Kafka internal topic.
     IsInternal: bool // v1+
     // Partitions contains metadata about partitions for a topic.
@@ -74,6 +81,9 @@ MetadataResponse =>
       //
       // REPLICA_NOT_AVAILABLE is returned in v0 responses if any replica is
       // unavailable.
+      //
+      // UNKNOWN_TOPIC_ID is returned if using a topic ID and the ID does not
+      // exist.
       ErrorCode: int16
       // Partition is a partition number for a topic.
       Partition: int32

generate/definitions/04_leader_and_isr

@@ -1,6 +1,6 @@
 // LeaderAndISRRequestTopicPartition is a common struct that is used across
 // different versions of LeaderAndISRRequest.
-LeaderAndISRRequestTopicPartition => not top level, no encoding
+LeaderAndISRRequestTopicPartition => not top level, no encoding, flexible v4+
   Topic: string // v0-v1
   Partition: int32
   ControllerEpoch: int32
@@ -13,6 +13,12 @@ LeaderAndISRRequestTopicPartition => not top level, no encoding
   RemovingReplicas: [int32] // v3+
   IsNew: bool // v1+
 
+// LeaderAndISRResponseTopicPartition is a common struct that is used across
+// different versions of LeaderAndISRResponse.
+LeaderAndISRResponseTopicPartition => not top level, no encoding, flexible v4+
+  Topic: string // v0-v4
+  Partition: int32
+  ErrorCode: int16
 
 // LeaderAndISRRequest is an advanced request that controller brokers use
 // to broadcast state to other brokers. Manually using this request is a
@@ -24,13 +30,15 @@ LeaderAndISRRequestTopicPartition => not top level, no encoding
 // Kafka 1.0.0 introduced version 1. Kafka 2.2.0 introduced version 2, proposed
 // in KIP-380, which changed the layout of the struct to be more memory
 // efficient. Kafka 2.4.0 introduced version 3 with KIP-455.
-LeaderAndISRRequest => key 4, max version 4, flexible v4+
+LeaderAndISRRequest => key 4, max version 5, flexible v4+
   ControllerID: int32
   ControllerEpoch: int32
   BrokerEpoch: int64(-1) // v2+
+  Type: int8 // v5+
   PartitionStates: [LeaderAndISRRequestTopicPartition] // v0-v1
   TopicStates: [=>] // v2+
     Topic: string
+    TopicID: uuid // v5+
     PartitionStates: [LeaderAndISRRequestTopicPartition]
   LiveLeaders: [=>]
     BrokerID: int32
@@ -40,7 +48,7 @@ LeaderAndISRRequest => key 4, max version 4, flexible v4+
 // LeaderAndISRResponse is returned from a LeaderAndISRRequest.
 LeaderAndISRResponse =>
   ErrorCode: int16
-  Partitions: [=>]
-    Topic: string
-    Partition: int32
-    ErrorCode: int16
+  Partitions: [LeaderAndISRResponseTopicPartition] // v0-v4
+  Topics: [=>] // v5+
+    TopicID: uuid
+    Partitions: [LeaderAndISRResponseTopicPartition]

generate/definitions/06_update_metadata

@@ -1,4 +1,4 @@
-UpdateMetadataRequestTopicPartition => not top level, no encoding
+UpdateMetadataRequestTopicPartition => not top level, no encoding, flexible v6+
   Topic: string // v0-v4
   Partition: int32
   ControllerEpoch: int32
@@ -19,13 +19,14 @@ UpdateMetadataRequestTopicPartition => not top level, no encoding
 //
 // Kafka 2.2.0 introduced version 5, proposed in KIP-380, which changed the
 // layout of the struct to be more memory efficient.
-UpdateMetadataRequest => key 6, max version 6, flexible v6+
+UpdateMetadataRequest => key 6, max version 7, flexible v6+
   ControllerID: int32
   ControllerEpoch: int32
   BrokerEpoch: int64(-1) // v5+
   PartitionStates: [UpdateMetadataRequestTopicPartition] // v0-v4
   TopicStates: [=>] // v5+
     Topic: string
+    TopicID: uuid // v7+
     PartitionStates: [UpdateMetadataRequestTopicPartition]
   LiveBrokers: [=>]
     ID: int32

pkg/kversion/kversion.go

@@ -454,5 +454,12 @@ func Tip() Versions {
 	v[48]++ // 1 describe client quotas
 	v[49]++ // 1 alter client quotas
 
+	// KAFKA-10547 5c921afa4a593478f7d1c49e5db9d787558d0d5e KIP-516
+	v[3]++ // 10 metadata
+	v[6]++ // 7 update metadata
+
+	// KAFKA-10545 1dd1e7f945d7a8c1dc177223cd88800680f1ff46 KIP-516
+	v[4]++ // 5 leader and isr
+
 	return v
 }