types/descriptorpb: regenerate using latest protobuf v22.0 release

This updates all generated code to match the contents of the latest v22.0 release of Protobuf. This involved a couple of changes to the script that does the sync'ing: 1. The new Protobuf version no longer includes autoconf configuration and instead requires using bazel to build things. 2. The new Protobuf release does not have an artifact named "protobuf-all-${VERSION}.tar.gz", but the one named "protobuf-${VERSION}.tar.gz" has all of the sources and was sufficient for the regenerate.bash script to complete. This change does NOT regenerate the protos related to benchmarks. The Protobuf repo no longer includes benchmarks. The CL removing them says they are superceded by google/fleetbench: protocolbuffers/protobuf@83c499d But that project's proto benchmark files are very different: https://github.com/google/fleetbench/tree/main/fleetbench/proto So I commented out those steps in the generation code since the benchmarks will need some work to reconcile with fleetbench. This code adds known failing tests for conformance. New test cases were added in protocolbuffers/protobuf#9534, but the Go protojson package does not behave according to the new tests. There is an existing issue in GitHub about this: golang/protobuf#1208 Change-Id: Iad796ec7889bc2a74b58db5224facf850cd1a1cd Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/469255 Reviewed-by: Michael Stapelberg <[email protected]> Reviewed-by: Damien Neil <[email protected]>
protocolbuffers · Feb 22, 2023 · bc1253a · bc1253a
1 parent 246af0d
commit bc1253a
Show file tree

Hide file tree

Showing 37 changed files with 1,582 additions and 1,033 deletions.
diff --git a/cmd/protoc-gen-go/testdata/comments/comments.pb.go b/cmd/protoc-gen-go/testdata/comments/comments.pb.go
diff --git a/cmd/protoc-gen-go/testdata/fieldnames/fieldnames.pb.go b/cmd/protoc-gen-go/testdata/fieldnames/fieldnames.pb.go
diff --git a/cmd/protoc-gen-go/testdata/import_public/sub/a.pb.go b/cmd/protoc-gen-go/testdata/import_public/sub/a.pb.go
diff --git a/cmd/protoc-gen-go/testdata/issue780_oneof_conflict/test.pb.go b/cmd/protoc-gen-go/testdata/issue780_oneof_conflict/test.pb.go
diff --git a/cmd/protoc-gen-go/testdata/proto2/fields.pb.go b/cmd/protoc-gen-go/testdata/proto2/fields.pb.go
diff --git a/integration_test.go b/integration_test.go
@@ -35,7 +35,7 @@ var (
 	regenerate   = flag.Bool("regenerate", false, "regenerate files")
 	buildRelease = flag.Bool("buildRelease", false, "build release binaries")
 
-	protobufVersion = "21.5"
+	protobufVersion = "22.0"
 	protobufSHA256  = "" // ignored if protobufVersion is a git hash
 
 	golangVersions = func() []string {
@@ -45,7 +45,7 @@ var (
 		default:
 			vers = []string{"1.13.15", "1.14.15", "1.15.15"}
 		}
-		return append(vers, "1.16.15", "1.17.12", "1.18.4")
+		return append(vers, "1.16.15", "1.17.13", "1.18.10", "1.19.6")
 	}()
 	golangLatest = golangVersions[len(golangVersions)-1]
 
@@ -244,23 +244,20 @@ func mustInitDeps(t *testing.T) {
 	protobufPath = startWork("protobuf-" + protobufVersion)
 	if _, err := os.Stat(protobufPath); err != nil {
 		fmt.Printf("download %v\n", filepath.Base(protobufPath))
-		if isCommit := strings.Trim(protobufVersion, "0123456789abcdef") == ""; isCommit {
-			command{Dir: testDir}.mustRun(t, "git", "clone", "https://github.com/protocolbuffers/protobuf", "protobuf-"+protobufVersion)
-			command{Dir: protobufPath}.mustRun(t, "git", "checkout", protobufVersion)
-		} else {
-			url := fmt.Sprintf("https://github.com/google/protobuf/releases/download/v%v/protobuf-all-%v.tar.gz", protobufVersion, protobufVersion)
-			downloadArchive(check, protobufPath, url, "protobuf-"+protobufVersion, protobufSHA256)
+		checkoutVersion := protobufVersion
+		if isCommit := strings.Trim(protobufVersion, "0123456789abcdef") == ""; !isCommit {
+			// release tags have "v" prefix
+			checkoutVersion = "v" + protobufVersion
 		}
+		command{Dir: testDir}.mustRun(t, "git", "clone", "https://github.com/protocolbuffers/protobuf", "protobuf-"+protobufVersion)
+		command{Dir: protobufPath}.mustRun(t, "git", "checkout", checkoutVersion)
 
 		fmt.Printf("build %v\n", filepath.Base(protobufPath))
-		command{Dir: protobufPath}.mustRun(t, "./autogen.sh")
-		command{Dir: protobufPath}.mustRun(t, "./configure")
-		command{Dir: protobufPath}.mustRun(t, "make")
-		command{Dir: filepath.Join(protobufPath, "conformance")}.mustRun(t, "make")
+		command{Dir: protobufPath}.mustRun(t, "bazel", "build", ":protoc", "//conformance:conformance_test_runner")
 	}
 	check(os.Setenv("PROTOBUF_ROOT", protobufPath)) // for generate-protos
-	registerBinary("conform-test-runner", filepath.Join(protobufPath, "conformance", "conformance-test-runner"))
-	registerBinary("protoc", filepath.Join(protobufPath, "src", "protoc"))
+	registerBinary("conform-test-runner", filepath.Join(protobufPath, "bazel-bin", "conformance", "conformance_test_runner"))
+	registerBinary("protoc", filepath.Join(protobufPath, "bazel-bin", "protoc"))
 	finishWork()
 
 	// Download each Go toolchain version.

diff --git a/internal/cmd/generate-protos/main.go b/internal/cmd/generate-protos/main.go
@@ -195,23 +195,30 @@ func generateRemoteProtos() {
 		{"src", "google/protobuf/test_messages_proto3.proto", "google.golang.org/protobuf/internal/testprotos/conformance;conformance"},
 
 		// Benchmark protos.
-		{"benchmarks", "benchmarks.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks;benchmarks"},
-		{"benchmarks", "datasets/google_message1/proto2/benchmark_message1_proto2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message1/proto2;proto2"},
-		{"benchmarks", "datasets/google_message1/proto3/benchmark_message1_proto3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message1/proto3;proto3"},
-		{"benchmarks", "datasets/google_message2/benchmark_message2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message2;google_message2"},
-		{"benchmarks", "datasets/google_message3/benchmark_message3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
-		{"benchmarks", "datasets/google_message3/benchmark_message3_1.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
-		{"benchmarks", "datasets/google_message3/benchmark_message3_2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
-		{"benchmarks", "datasets/google_message3/benchmark_message3_3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
-		{"benchmarks", "datasets/google_message3/benchmark_message3_4.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
-		{"benchmarks", "datasets/google_message3/benchmark_message3_5.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
-		{"benchmarks", "datasets/google_message3/benchmark_message3_6.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
-		{"benchmarks", "datasets/google_message3/benchmark_message3_7.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
-		{"benchmarks", "datasets/google_message3/benchmark_message3_8.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
-		{"benchmarks", "datasets/google_message4/benchmark_message4.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
-		{"benchmarks", "datasets/google_message4/benchmark_message4_1.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
-		{"benchmarks", "datasets/google_message4/benchmark_message4_2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
-		{"benchmarks", "datasets/google_message4/benchmark_message4_3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
+		// TODO: The protobuf repo no longer includes benchmarks.
+		//       CL removing them says they are superceded by google/fleetbench:
+		//         https://github.com/protocolbuffers/protobuf/commit/83c499de86224538e5d59adc3d0fa7fdb45b2c72
+		//       But that project's proto benchmark files are very different:
+		//         https://github.com/google/fleetbench/tree/main/fleetbench/proto
+		//       For now, commenting these out until benchmarks in this repo can be
+		//       reconciled with new fleetbench stuff.
+		//{"benchmarks", "benchmarks.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks;benchmarks"},
+		//{"benchmarks", "datasets/google_message1/proto2/benchmark_message1_proto2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message1/proto2;proto2"},
+		//{"benchmarks", "datasets/google_message1/proto3/benchmark_message1_proto3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message1/proto3;proto3"},
+		//{"benchmarks", "datasets/google_message2/benchmark_message2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message2;google_message2"},
+		//{"benchmarks", "datasets/google_message3/benchmark_message3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
+		//{"benchmarks", "datasets/google_message3/benchmark_message3_1.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
+		//{"benchmarks", "datasets/google_message3/benchmark_message3_2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
+		//{"benchmarks", "datasets/google_message3/benchmark_message3_3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
+		//{"benchmarks", "datasets/google_message3/benchmark_message3_4.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
+		//{"benchmarks", "datasets/google_message3/benchmark_message3_5.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
+		//{"benchmarks", "datasets/google_message3/benchmark_message3_6.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
+		//{"benchmarks", "datasets/google_message3/benchmark_message3_7.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
+		//{"benchmarks", "datasets/google_message3/benchmark_message3_8.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message3;google_message3"},
+		//{"benchmarks", "datasets/google_message4/benchmark_message4.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
+		//{"benchmarks", "datasets/google_message4/benchmark_message4_1.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
+		//{"benchmarks", "datasets/google_message4/benchmark_message4_2.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
+		//{"benchmarks", "datasets/google_message4/benchmark_message4_3.proto", "google.golang.org/protobuf/internal/testprotos/benchmarks/datasets/google_message4;google_message4"},
 	}
 
 	opts := "module=" + modulePath

diff --git a/internal/conformance/conformance_test.go b/internal/conformance/conformance_test.go
@@ -40,7 +40,7 @@ func Test(t *testing.T) {
 	if !*execute {
 		t.SkipNow()
 	}
-	binPath := filepath.Join(*protoRoot, "conformance", "conformance-test-runner")
+	binPath := filepath.Join(*protoRoot, "bazel-bin", "conformance", "conformance_test_runner")
 	cmd := exec.Command(binPath,
 		"--failure_list", "failing_tests.txt",
 		"--text_format_failure_list", "failing_tests_text_format.txt",

diff --git a/internal/conformance/failing_tests.txt b/internal/conformance/failing_tests.txt
@@ -0,0 +1,3 @@
+Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput
+Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInOptionalField.ProtobufOutput
+Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInRepeatedField.ProtobufOutput
diff --git a/internal/filedesc/build_test.go b/internal/filedesc/build_test.go
@@ -53,6 +53,8 @@ func TestInit(t *testing.T) {
 		// The protoreflect descriptors don't include source info.
 		descPkg.Append("FileDescriptorProto.source_code_info"): true,
 		descPkg.Append("FileDescriptorProto.syntax"):           true,
+		// Nothing is using edition yet.
+		descPkg.Append("FileDescriptorProto.edition"): true,
 
 		// Impossible to test proto3 optional in a proto2 file.
 		descPkg.Append("FieldDescriptorProto.proto3_optional"): true,