-
Notifications
You must be signed in to change notification settings - Fork 3.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
110377: pkg/server: add an extra server startup guardrail r=knz a=healthy-pod This code change prevents a SQL server from starting if its minimum supported binary version is greater than the tenant active version. Release note: None Epic: CRDB-26691 111143: cli: add --experimental-secondary-cache flag r=RaduBerinde a=itsbilal This change adds a new `--experimental-secondary-cache` flag for use with `--experimental-shared-storage` to enable the use of a secondary cache to speed up reads of objects in shared storage. This option sets the max cache size for each store using disaggregated storage; for per-store granularity, pebble options can also be used. Epic: none Release note: None 111173: sql: add some tests for lookup join when eq cols are key r=yuzefovich a=yuzefovich These tests are "regression tests" for #108489 if it were to be implemented. Epic: None Release note: None 111325: roachtest: add test to ruby-pg blocklist r=fqazi a=andyyang890 Fixes #111116 Release note: None 111384: process-bep-file: create binary for fetching test results from EngFlow r=healthy-pod a=rickystewart I'll extend this to additionally allow posting GitHub issues. Part of: DEVINF-871 Epic: CRDB-8308 Release note: None 111433: kv: add shared, replicated, and shared-replicated locks to TestEvaluateBatch r=nvanbenschoten a=nvanbenschoten Informs #91545. Informs #100193. This commit extends TestEvaluateBatch to include shared, replicated, and shared-replicated lock acquisition using Get, Scan, and ReverseScan requests. Release note: None 111434: roachtest: automatically profile CPU in restore tests r=msbutler a=pavelkalinnikov See https://www.cockroachlabs.com/docs/v23.1/automatic-cpu-profiler. NB: the `server.cpu_profile.enabled` setting is not used because it was recently removed in #107717. It should be used if this change is backported. Touches #111160, #111159 Epic: none Release note: none Co-authored-by: healthy-pod <[email protected]> Co-authored-by: Bilal Akhtar <[email protected]> Co-authored-by: Yahor Yuzefovich <[email protected]> Co-authored-by: Andy Yang <[email protected]> Co-authored-by: Ricky Stewart <[email protected]> Co-authored-by: Nathan VanBenschoten <[email protected]> Co-authored-by: Pavel Kalinnikov <[email protected]>
- Loading branch information
Showing
13 changed files
with
654 additions
and
86 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") | ||
|
|
||
| go_library( | ||
| name = "process-bep-file_lib", | ||
| srcs = ["main.go"], | ||
| importpath = "github.com/cockroachdb/cockroach/pkg/cmd/bazci/process-bep-file", | ||
| visibility = ["//visibility:private"], | ||
| deps = [ | ||
| "//pkg/build/bazel/bes", | ||
| "@com_github_golang_protobuf//proto:go_default_library", | ||
| "@org_golang_x_net//http2", | ||
| ], | ||
| ) | ||
|
|
||
| go_binary( | ||
| name = "process-bep-file", | ||
| embed = [":process-bep-file_lib"], | ||
| visibility = ["//visibility:public"], | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,191 @@ | ||
| // Copyright 2023 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
|
|
||
| //go:build bazel | ||
| // +build bazel | ||
|
|
||
| // process-bep-file is a binary to parse test results from a "build event | ||
| // protocol" binary file, as constructed by | ||
| // `bazel ... --build_event_binary_file`. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "crypto/tls" | ||
| "flag" | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
| "os" | ||
| "strings" | ||
| "sync" | ||
|
|
||
| bes "github.com/cockroachdb/cockroach/pkg/build/bazel/bes" | ||
| //lint:ignore SA1019 | ||
| gproto "github.com/golang/protobuf/proto" | ||
| "golang.org/x/net/http2" | ||
| ) | ||
|
|
||
| type testResultWithMetadata struct { | ||
| run, shard, attempt int32 | ||
| testResult *bes.TestResult | ||
| } | ||
|
|
||
| type testResultWithXml struct { | ||
| label string | ||
| run, shard, attempt int32 | ||
| testResult *bes.TestResult | ||
| testXml string | ||
| err error | ||
| } | ||
|
|
||
| var ( | ||
| eventStreamFile = flag.String("eventsfile", "", "eventstream file produced by bazel build --build_event_binary_file") | ||
| tlsClientCert = flag.String("cert", "", "TLS client certificate for accessing EngFlow, probably a .crt file") | ||
| tlsClientKey = flag.String("key", "", "TLS client key for accessing EngFlow") | ||
| ) | ||
|
|
||
| func getHttpClient() (*http.Client, error) { | ||
| cer, err := tls.LoadX509KeyPair(*tlsClientCert, *tlsClientKey) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| config := &tls.Config{ | ||
| Certificates: []tls.Certificate{cer}, | ||
| } | ||
| transport := &http2.Transport{ | ||
| TLSClientConfig: config, | ||
| } | ||
| httpClient := &http.Client{ | ||
| Transport: transport, | ||
| } | ||
| return httpClient, nil | ||
| } | ||
|
|
||
| func downloadTestXml(client *http.Client, uri string) (string, error) { | ||
| url := strings.ReplaceAll(uri, "bytestream://", "https://") | ||
| url = strings.ReplaceAll(url, "/blobs/", "/api/v0/blob/") | ||
| resp, err := client.Get(url) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| defer func() { _ = resp.Body.Close() }() | ||
| xml, err := io.ReadAll(resp.Body) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| return string(xml), nil | ||
| } | ||
|
|
||
| func fetchTestXml( | ||
| httpClient *http.Client, | ||
| label string, | ||
| testResult testResultWithMetadata, | ||
| ch chan testResultWithXml, | ||
| wg *sync.WaitGroup, | ||
| ) { | ||
| defer wg.Done() | ||
| for _, output := range testResult.testResult.TestActionOutput { | ||
| if output.Name == "test.xml" { | ||
| xml, err := downloadTestXml(httpClient, output.GetUri()) | ||
| ch <- testResultWithXml{ | ||
| label: label, | ||
| run: testResult.run, | ||
| shard: testResult.shard, | ||
| attempt: testResult.attempt, | ||
| testXml: xml, | ||
| err: err, | ||
| testResult: testResult.testResult, | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func process() error { | ||
| httpClient, err := getHttpClient() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| content, err := os.ReadFile(*eventStreamFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| buf := gproto.NewBuffer(content) | ||
| testResults := make(map[string][]testResultWithMetadata) | ||
| fullTestResults := make(map[string][]testResultWithXml) | ||
| for { | ||
| var event bes.BuildEvent | ||
| err := buf.DecodeMessage(&event) | ||
| if err != nil { | ||
| // This is probably OK: just no more stuff left in the buffer. | ||
| break | ||
| } | ||
| switch id := event.Id.Id.(type) { | ||
| case *bes.BuildEventId_TestResult: | ||
| res := testResultWithMetadata{ | ||
| run: id.TestResult.Run, | ||
| shard: id.TestResult.Shard, | ||
| attempt: id.TestResult.Attempt, | ||
| testResult: event.GetTestResult(), | ||
| } | ||
| testResults[id.TestResult.Label] = append(testResults[id.TestResult.Label], res) | ||
| } | ||
| } | ||
| unread := buf.Unread() | ||
| if len(unread) != 0 { | ||
| return fmt.Errorf("didn't read entire file: %d bytes remaining", len(unread)) | ||
| } | ||
|
|
||
| // Download test xml's. | ||
| ch := make(chan testResultWithXml) | ||
| var wg sync.WaitGroup | ||
| for label, results := range testResults { | ||
| for _, result := range results { | ||
| wg.Add(1) | ||
| go fetchTestXml(httpClient, label, result, ch, &wg) | ||
| } | ||
| } | ||
| go func(wg *sync.WaitGroup) { | ||
| wg.Wait() | ||
| close(ch) | ||
| }(&wg) | ||
|
|
||
| // Collect test xml's. | ||
| for result := range ch { | ||
| fullTestResults[result.label] = append(fullTestResults[result.label], result) | ||
| if result.err != nil { | ||
| fmt.Printf("got error downloading test XML for result %+v; got error %+v", result, result.err) | ||
| } | ||
| } | ||
|
|
||
| // TODO: handle results | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func main() { | ||
| flag.Parse() | ||
| if *eventStreamFile == "" { | ||
| fmt.Println("must provide -eventsfile") | ||
| os.Exit(1) | ||
| } | ||
| if *tlsClientCert == "" { | ||
| fmt.Println("must provide -cert") | ||
| os.Exit(1) | ||
| } | ||
| if *tlsClientKey == "" { | ||
| fmt.Println("must provide -key") | ||
| os.Exit(1) | ||
| } | ||
| if err := process(); err != nil { | ||
| fmt.Printf("ERROR: %+v", err) | ||
| os.Exit(1) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.