From c9599e98e2ff9a5d25f78ed78a8794711781ddee Mon Sep 17 00:00:00 2001 From: Aditya Maru Date: Wed, 19 May 2021 09:40:33 +0100 Subject: [PATCH 1/3] jobs: move trace_id from payload to progress The trace_id associated with a job is updated on every resumption of the job. If the job is paused and resumed several times, we don't want to rewrite the payload every single time. Thus, we move the field to the job progress which is expected to be rewritten frequently. Also addresses a comment from #65322. Release note: None --- pkg/jobs/adopt.go | 10 +- pkg/jobs/jobs.go | 2 +- pkg/jobs/jobspb/jobs.pb.go | 700 ++++++++++++++++++------------------- pkg/jobs/jobspb/jobs.proto | 4 +- pkg/sql/crdb_internal.go | 2 +- 5 files changed, 359 insertions(+), 359 deletions(-) diff --git a/pkg/jobs/adopt.go b/pkg/jobs/adopt.go index a7b90c0edd0f..e9ac7c935c0d 100644 --- a/pkg/jobs/adopt.go +++ b/pkg/jobs/adopt.go @@ -255,17 +255,15 @@ func (r *Registry) runJob( // // A new root span will be created on every resumption of the job. var spanOptions []tracing.SpanOption - if tj, ok := resumer.(TraceableJob); ok { - if tj.ForceRealSpan() { - spanOptions = append(spanOptions, tracing.WithForceRealSpan()) - } + if _, ok := resumer.(TraceableJob); ok { + spanOptions = append(spanOptions, tracing.WithForceRealSpan()) } ctx, span = r.settings.Tracer.StartSpanCtx(ctx, spanName, spanOptions...) defer span.Finish() if err := job.Update(ctx, nil /* txn */, func(txn *kv.Txn, md JobMetadata, ju *JobUpdater) error { - md.Payload.TraceID = span.TraceID() - ju.UpdatePayload(md.Payload) + md.Progress.TraceID = span.TraceID() + ju.UpdateProgress(md.Progress) return nil }); err != nil { return err diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go index 4af9f93b5ea8..23c4f2e8726e 100644 --- a/pkg/jobs/jobs.go +++ b/pkg/jobs/jobs.go @@ -97,7 +97,7 @@ type StartableJob struct { type TraceableJob interface { // ForceRealSpan forces the registry to create a real Span instead of a // low-overhead non-recordable noop span. - ForceRealSpan() bool + ForceRealSpan() } func init() { diff --git a/pkg/jobs/jobspb/jobs.pb.go b/pkg/jobs/jobspb/jobs.pb.go index 487451e589ad..bba93041a0dc 100644 --- a/pkg/jobs/jobspb/jobs.pb.go +++ b/pkg/jobs/jobspb/jobs.pb.go @@ -2099,7 +2099,6 @@ type Payload struct { // *Payload_NewSchemaChange // *Payload_Migration Details isPayload_Details `protobuf_oneof:"details"` - TraceID uint64 `protobuf:"varint,26,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` } func (m *Payload) Reset() { *m = Payload{} } @@ -2304,6 +2303,7 @@ type Progress struct { // *Progress_NewSchemaChange // *Progress_Migration Details isProgress_Details `protobuf_oneof:"details"` + TraceID uint64 `protobuf:"varint,21,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` } func (m *Progress) Reset() { *m = Progress{} } @@ -2629,326 +2629,326 @@ func init() { func init() { proto.RegisterFile("jobs/jobspb/jobs.proto", fileDescriptor_6c315f3a2536c4ef) } var fileDescriptor_6c315f3a2536c4ef = []byte{ - // 5096 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x5b, 0x4b, 0x70, 0x1b, 0x47, - 0x7a, 0xe6, 0x80, 0x20, 0x30, 0xf8, 0x09, 0x80, 0x83, 0x26, 0x25, 0xc1, 0x58, 0x5b, 0xe0, 0x62, - 0x2d, 0x5b, 0x92, 0x6d, 0xd0, 0x4b, 0xef, 0x7a, 0x6d, 0xad, 0x2d, 0x1b, 0x2f, 0x92, 0x00, 0xc5, - 0x87, 0x06, 0xa4, 0xfc, 0xd8, 0x78, 0x27, 0x83, 0x99, 0x26, 0x39, 0x21, 0x30, 0x03, 0x4d, 0x0f, - 0x24, 0x71, 0x53, 0x95, 0xa4, 0x76, 0x2b, 0x55, 0x5b, 0x3a, 0x25, 0x87, 0xcd, 0x21, 0x89, 0xaa, - 0xb6, 0x6a, 0x77, 0xab, 0x72, 0x48, 0x0e, 0xd9, 0x4a, 0x6d, 0x72, 0xc8, 0x29, 0xb9, 0xf8, 0x90, - 0x54, 0xed, 0xd1, 0xc9, 0x01, 0x49, 0xe8, 0x4b, 0x8e, 0xa9, 0xe4, 0x92, 0xd2, 0x29, 0xd5, 0x8f, - 0x19, 0x0c, 0xc0, 0x17, 0x28, 0xca, 0x9b, 0x8b, 0x84, 0xf9, 0xbb, 0xfb, 0xeb, 0xd7, 0xdf, 0xdf, - 0xff, 0xe8, 0x26, 0x5c, 0xfe, 0x1d, 0xa7, 0x45, 0x16, 0xe8, 0x3f, 0xdd, 0x16, 0xfb, 0xaf, 0xd8, - 0x75, 0x1d, 0xcf, 0x41, 0x2f, 0x18, 0x8e, 0xb1, 0xef, 0x3a, 0xba, 0xb1, 0x57, 0x24, 0xf7, 0xdb, - 0x45, 0x56, 0xc2, 0x6b, 0xe5, 0x2e, 0x61, 0xd7, 0x75, 0x5c, 0x5a, 0x9f, 0xff, 0xe0, 0x2d, 0x72, - 0x73, 0xbb, 0xce, 0xae, 0xc3, 0x7e, 0x2e, 0xd0, 0x5f, 0x42, 0x8a, 0x18, 0x46, 0xb7, 0xb5, 0x60, - 0xea, 0x9e, 0x2e, 0x64, 0x59, 0x5f, 0x66, 0x39, 0x6f, 0xec, 0x38, 0x6e, 0x47, 0xf7, 0x7c, 0x8c, - 0x6f, 0x90, 0xfb, 0xed, 0x05, 0x43, 0xf7, 0xf4, 0xb6, 0xb3, 0xbb, 0x60, 0x62, 0x62, 0x74, 0x5b, - 0x0b, 0xc4, 0x73, 0x7b, 0x86, 0xd7, 0x73, 0xb1, 0x29, 0x2a, 0xe5, 0x8f, 0xa9, 0xe4, 0x61, 0x5b, - 0xb7, 0x3d, 0x1f, 0xbf, 0xe7, 0x59, 0xed, 0x85, 0xbd, 0xb6, 0xb1, 0xe0, 0x59, 0x1d, 0x4c, 0x3c, - 0xbd, 0xd3, 0x15, 0x25, 0x5f, 0xa7, 0x4d, 0x89, 0xb1, 0x87, 0x3b, 0xba, 0xb1, 0xa7, 0xdb, 0xbb, - 0xd8, 0x5d, 0xe0, 0x7d, 0x18, 0xdd, 0x96, 0xa8, 0xf2, 0xb2, 0xd1, 0xee, 0x11, 0x0f, 0xbb, 0x0f, - 0xb0, 0x4b, 0x2c, 0xc7, 0x5e, 0x10, 0x9f, 0x9a, 0xf8, 0xe6, 0xb5, 0x0a, 0xbf, 0x0f, 0x53, 0x77, - 0xb0, 0x4e, 0x30, 0xfa, 0x14, 0xe2, 0xb6, 0x63, 0x62, 0xcd, 0x32, 0xb3, 0xd2, 0xbc, 0x74, 0x3d, - 0x55, 0x2e, 0x1d, 0xf6, 0xf3, 0xb1, 0x75, 0xc7, 0xc4, 0xf5, 0xea, 0xd3, 0x7e, 0xfe, 0xad, 0x5d, - 0xcb, 0xdb, 0xeb, 0xb5, 0x8a, 0x86, 0xd3, 0x59, 0x08, 0x56, 0xd4, 0x6c, 0x0d, 0x7e, 0x2f, 0x74, - 0xf7, 0x77, 0x17, 0xc4, 0x7a, 0x14, 0x79, 0x33, 0x35, 0x46, 0x11, 0xeb, 0x26, 0x9a, 0x83, 0x29, - 0xdc, 0x75, 0x8c, 0xbd, 0x6c, 0x64, 0x5e, 0xba, 0x3e, 0xa9, 0xf2, 0x8f, 0x5b, 0xd1, 0xff, 0xfc, - 0x69, 0x5e, 0x2a, 0xfc, 0x22, 0x02, 0x57, 0xca, 0xba, 0xb1, 0xdf, 0xeb, 0xd6, 0x6c, 0xc3, 0x3d, - 0xe8, 0x7a, 0x96, 0x63, 0x6f, 0xb0, 0x7f, 0x09, 0x52, 0x60, 0x72, 0x1f, 0x1f, 0xb0, 0xf1, 0x24, - 0x55, 0xfa, 0x13, 0xbd, 0x0f, 0xd1, 0x8e, 0x63, 0x62, 0x06, 0x94, 0x5e, 0xbc, 0x51, 0x3c, 0x71, - 0x73, 0x8b, 0x03, 0xb4, 0x35, 0xc7, 0xc4, 0x2a, 0x6b, 0x86, 0x5a, 0x20, 0xef, 0x77, 0x88, 0x66, - 0xd9, 0x3b, 0x4e, 0x76, 0x72, 0x5e, 0xba, 0x3e, 0xbd, 0x78, 0xeb, 0x14, 0x88, 0x13, 0x86, 0x55, - 0x5c, 0x5d, 0x6b, 0xd6, 0xed, 0x1d, 0xa7, 0x3c, 0x7d, 0xd8, 0xcf, 0xc7, 0xc5, 0x87, 0x1a, 0xdf, - 0xef, 0x10, 0xfa, 0x23, 0xb7, 0x01, 0xbe, 0x8c, 0x8e, 0xbf, 0xe7, 0x5a, 0x6c, 0xfc, 0x09, 0x95, - 0xfe, 0x44, 0xaf, 0x03, 0xc2, 0x1c, 0x0f, 0x9b, 0x1a, 0xd5, 0x24, 0x8d, 0x4e, 0x30, 0xc2, 0x26, - 0xa8, 0x04, 0x25, 0x55, 0xdd, 0xd3, 0x57, 0xf1, 0x01, 0x5f, 0x21, 0xb1, 0x4e, 0x7f, 0x30, 0x09, - 0xe9, 0xc1, 0x50, 0x18, 0xfc, 0x0a, 0xc4, 0x98, 0x0a, 0x60, 0xd6, 0x43, 0x7a, 0xf1, 0xcd, 0xb1, - 0x96, 0x83, 0x36, 0x2d, 0x36, 0x59, 0x3b, 0x55, 0xb4, 0x47, 0x08, 0xa2, 0x44, 0x6f, 0x7b, 0x62, - 0x20, 0xec, 0x37, 0xfa, 0x33, 0x09, 0xe6, 0x47, 0x47, 0x54, 0x3e, 0x58, 0x5d, 0x6b, 0xae, 0xe9, - 0x54, 0x8f, 0x56, 0xf1, 0x41, 0xbd, 0x9a, 0x9d, 0x9c, 0x9f, 0xbc, 0x3e, 0xbd, 0xb8, 0x31, 0x7e, - 0xc7, 0xb5, 0x33, 0x10, 0x6b, 0xb6, 0xe7, 0x1e, 0xa8, 0x67, 0x76, 0x9c, 0x6b, 0xc2, 0xb5, 0xb1, - 0xa0, 0xc2, 0x3a, 0x94, 0xe0, 0x3a, 0x34, 0x07, 0x53, 0x0f, 0xf4, 0x76, 0x0f, 0x8b, 0xd9, 0xf2, - 0x8f, 0x5b, 0x91, 0x77, 0xa4, 0xc2, 0x15, 0x88, 0xf1, 0x85, 0x41, 0x29, 0x48, 0x94, 0x6a, 0xcd, - 0xc5, 0x6f, 0xbf, 0xbd, 0x5c, 0x59, 0x53, 0x26, 0xc4, 0x16, 0xfc, 0x4a, 0x82, 0xcb, 0x4d, 0xcf, - 0xc5, 0x7a, 0xa7, 0x6e, 0xef, 0x62, 0x42, 0xe7, 0x54, 0xc5, 0x9e, 0x6e, 0xb5, 0x09, 0xba, 0x06, - 0x69, 0xc2, 0x4a, 0x34, 0xdd, 0x34, 0x5d, 0x4c, 0x88, 0xe8, 0x30, 0xc5, 0xa5, 0x25, 0x2e, 0x44, - 0xdf, 0x84, 0x28, 0xe9, 0xea, 0x36, 0xeb, 0x79, 0x7a, 0xf1, 0x4a, 0x68, 0xd9, 0xfc, 0x93, 0xd3, - 0xec, 0xea, 0x76, 0x39, 0xfa, 0x79, 0x3f, 0x3f, 0xa1, 0xb2, 0xaa, 0xa8, 0x0c, 0x40, 0x3c, 0xdd, - 0xf5, 0x34, 0x4a, 0x01, 0x42, 0x69, 0x5f, 0x0a, 0x35, 0xa4, 0x14, 0x51, 0xdc, 0x6b, 0x1b, 0xc5, - 0x2d, 0x9f, 0x22, 0x44, 0xf3, 0x04, 0x6b, 0x46, 0xa5, 0x05, 0x1d, 0xae, 0x8c, 0x8c, 0x7b, 0xd3, - 0x75, 0x76, 0xd9, 0x88, 0x96, 0x20, 0x69, 0xf4, 0x3c, 0xe7, 0x01, 0x76, 0x79, 0x07, 0xd2, 0xf8, - 0x1d, 0x4c, 0x8b, 0x86, 0xac, 0x8b, 0xbf, 0x8f, 0x41, 0x8a, 0x9f, 0x17, 0x7f, 0x49, 0x86, 0x07, - 0x2e, 0x3d, 0xcb, 0xc0, 0xd1, 0x6d, 0x90, 0xb1, 0x6d, 0x72, 0x84, 0xc8, 0xf8, 0x08, 0x71, 0x6c, - 0x9b, 0xac, 0xfd, 0x0b, 0xfc, 0x00, 0xd2, 0x55, 0x4b, 0x94, 0xe3, 0x87, 0xfd, 0xfc, 0xe4, 0xb6, - 0x5a, 0xe7, 0x27, 0xf1, 0x47, 0x12, 0xcc, 0xf6, 0x5c, 0x8b, 0x68, 0xad, 0x03, 0xad, 0xed, 0x18, - 0x7a, 0xdb, 0xf2, 0x0e, 0xb4, 0xfd, 0x07, 0xd9, 0x29, 0xa6, 0xd1, 0xb7, 0xcf, 0xa4, 0x05, 0x31, - 0xcd, 0xe2, 0xb6, 0x6b, 0x91, 0xf2, 0xc1, 0x1d, 0x81, 0xb0, 0xfa, 0x80, 0x69, 0x5d, 0x79, 0xee, - 0xb0, 0x9f, 0x57, 0xb6, 0xd5, 0x7a, 0xb8, 0xe8, 0x9e, 0xaa, 0xf4, 0x46, 0x2a, 0xa3, 0xf7, 0x20, - 0x67, 0xe2, 0xae, 0x8b, 0x0d, 0x9d, 0x12, 0x42, 0x8b, 0x21, 0x6b, 0x1d, 0xdd, 0xb6, 0x76, 0x30, - 0xf1, 0xb2, 0x51, 0xa6, 0xa0, 0xd9, 0x41, 0x0d, 0xde, 0xf5, 0x9a, 0x28, 0x47, 0x7a, 0xc0, 0x26, - 0x96, 0x63, 0x6b, 0x0e, 0xa7, 0xa7, 0x6c, 0x8c, 0x2d, 0xd4, 0xe2, 0xf9, 0x89, 0x4d, 0xcd, 0xe0, - 0x23, 0x14, 0xac, 0xc2, 0x4c, 0xa8, 0x0b, 0x46, 0x9c, 0x09, 0x86, 0x7f, 0x63, 0xec, 0x33, 0xaf, - 0xa6, 0xf1, 0x30, 0x6f, 0xfd, 0xb1, 0x04, 0x39, 0x6a, 0x7d, 0xb0, 0x41, 0x27, 0x1d, 0x98, 0x36, - 0xcd, 0xc5, 0x86, 0xe3, 0x9a, 0xd9, 0x38, 0x9d, 0x75, 0xb9, 0xf9, 0xaf, 0xe3, 0x1a, 0x1d, 0x66, - 0x24, 0x7b, 0x3d, 0xcb, 0x2c, 0x6e, 0x6f, 0xd7, 0xab, 0x87, 0xfd, 0x7c, 0x76, 0xd3, 0x07, 0x0f, - 0x14, 0x43, 0x65, 0xd0, 0x6a, 0xb6, 0x7b, 0x42, 0x09, 0x7a, 0x07, 0xd2, 0x86, 0xd3, 0x6e, 0x63, - 0x83, 0xcd, 0x73, 0x5b, 0xad, 0x67, 0x65, 0xa6, 0x34, 0x99, 0xc3, 0x7e, 0x3e, 0x55, 0x09, 0x4a, - 0xa8, 0xfa, 0xa4, 0x8c, 0xf0, 0x67, 0xae, 0x02, 0x97, 0x8e, 0xd5, 0x81, 0xb3, 0x98, 0x27, 0x11, - 0x66, 0x1e, 0x05, 0xd2, 0x7c, 0x53, 0xfc, 0x83, 0x59, 0xf8, 0x07, 0x04, 0x69, 0x15, 0x13, 0xcf, - 0x71, 0xb1, 0x7f, 0xa2, 0xc2, 0xa7, 0x21, 0xfa, 0x0c, 0xa7, 0xe1, 0x97, 0x12, 0xcc, 0x52, 0x37, - 0xc3, 0xb5, 0xba, 0x9e, 0xe3, 0x6a, 0x2e, 0x7e, 0xe8, 0x5a, 0x1e, 0x26, 0xd9, 0x08, 0x53, 0xf9, - 0xd2, 0x29, 0x1b, 0x3a, 0x3c, 0x90, 0x62, 0x35, 0x00, 0x51, 0x05, 0x06, 0xd7, 0xfa, 0xdb, 0x3f, - 0xfc, 0xb7, 0xfc, 0xad, 0xb1, 0xf6, 0xec, 0xa8, 0xe7, 0x53, 0xac, 0x57, 0x55, 0x64, 0x1e, 0x01, - 0x46, 0x2f, 0x42, 0x94, 0x9e, 0x1a, 0x66, 0x69, 0x12, 0x65, 0xf9, 0xb0, 0x9f, 0x8f, 0xd2, 0x73, - 0xa5, 0x32, 0x29, 0xf2, 0x60, 0x4e, 0x1c, 0x9a, 0xe0, 0x0c, 0x33, 0x1d, 0x8d, 0xb3, 0x29, 0xbd, - 0x37, 0xfe, 0x94, 0xf8, 0xea, 0xfb, 0x5b, 0xc8, 0xcc, 0x3b, 0x5f, 0x3d, 0xd4, 0x3a, 0x52, 0x82, - 0x36, 0x21, 0x4d, 0x6d, 0x77, 0x4b, 0x27, 0x58, 0xa3, 0x43, 0x26, 0x59, 0x85, 0xf5, 0x37, 0x7a, - 0x26, 0xc8, 0xfd, 0x36, 0xad, 0x53, 0xac, 0x8a, 0xca, 0xa1, 0x75, 0x4b, 0x99, 0x21, 0x19, 0x41, - 0xcb, 0x30, 0xed, 0xe9, 0xad, 0xb6, 0x0f, 0xc7, 0x49, 0xe8, 0x95, 0x13, 0xe0, 0xb6, 0x68, 0xcd, - 0x10, 0x16, 0x78, 0xbe, 0x80, 0xa0, 0x2a, 0x80, 0x77, 0xd0, 0xf5, 0x71, 0xd2, 0x0c, 0xe7, 0xda, - 0x49, 0x38, 0x07, 0xdd, 0x30, 0x4c, 0xc2, 0x13, 0xdf, 0x04, 0x35, 0x20, 0xc9, 0x9d, 0x4b, 0x81, - 0x33, 0xc3, 0x70, 0x5e, 0x3d, 0x01, 0x87, 0xd9, 0x4c, 0x3d, 0x84, 0x34, 0x4d, 0x02, 0x09, 0x41, - 0x25, 0x88, 0x73, 0xa7, 0x96, 0x64, 0x53, 0x0c, 0xe6, 0xeb, 0x27, 0x0d, 0x87, 0xd5, 0x0a, 0x2d, - 0xbd, 0xdf, 0x0e, 0x2d, 0xc0, 0x34, 0x35, 0x34, 0xae, 0x65, 0x62, 0xcd, 0x6c, 0x31, 0x82, 0x4b, - 0x94, 0xd3, 0x87, 0xfd, 0x3c, 0x6c, 0x08, 0x71, 0xb5, 0xac, 0x82, 0x5f, 0xa5, 0xda, 0x42, 0xaf, - 0x41, 0xa6, 0xeb, 0xe2, 0xae, 0xee, 0x62, 0xcd, 0x70, 0x3a, 0xdd, 0x36, 0xf6, 0xb0, 0xc9, 0x0e, - 0xb4, 0xac, 0x2a, 0xa2, 0xa0, 0xe2, 0xcb, 0xb9, 0xed, 0xd6, 0x3d, 0xea, 0x16, 0x12, 0xec, 0xd2, - 0x9a, 0x09, 0x56, 0x33, 0xc5, 0xa4, 0x75, 0x21, 0x44, 0x07, 0x70, 0x99, 0x1c, 0x10, 0x0f, 0x77, - 0x34, 0xb6, 0xdc, 0x44, 0xeb, 0x58, 0xbb, 0x2e, 0x25, 0xe5, 0x6c, 0x86, 0x4d, 0xab, 0x32, 0xbe, - 0xb2, 0x35, 0x19, 0x0e, 0xdb, 0x46, 0xb2, 0x26, 0x50, 0xb8, 0xe3, 0x33, 0x47, 0x8e, 0x29, 0x42, - 0x6f, 0xc1, 0xa5, 0xc1, 0xc9, 0x20, 0x5a, 0xb7, 0xd7, 0x6a, 0x5b, 0x64, 0x0f, 0x9b, 0x59, 0x60, - 0x03, 0x9d, 0x0b, 0x15, 0x6e, 0xfa, 0x65, 0xe8, 0x60, 0xe8, 0xb0, 0x1b, 0x74, 0x75, 0xf4, 0x5d, - 0x9c, 0x9d, 0x9e, 0x97, 0xae, 0x4f, 0x95, 0x57, 0x9e, 0xf6, 0xf3, 0xd5, 0xb1, 0x4f, 0x2a, 0xc1, - 0x9d, 0x05, 0xcf, 0xc5, 0x38, 0x74, 0xf0, 0x2b, 0x02, 0x2f, 0x7c, 0x66, 0x7d, 0x19, 0x52, 0x01, - 0x06, 0x94, 0x9f, 0x4d, 0x3e, 0xb3, 0x3d, 0x0a, 0xa1, 0x20, 0x1b, 0x90, 0x8b, 0x1f, 0xe8, 0x6d, - 0xcb, 0xd4, 0x3d, 0xac, 0x59, 0xb6, 0x89, 0x1f, 0x61, 0x92, 0x45, 0x6c, 0xe9, 0xdf, 0x1d, 0x7f, - 0xe9, 0xd5, 0x00, 0xa3, 0x4e, 0x21, 0x84, 0xa6, 0x65, 0xdc, 0x61, 0x31, 0x26, 0xb9, 0xff, 0x91, - 0x20, 0x73, 0x84, 0xe7, 0xd0, 0x16, 0x44, 0x82, 0x00, 0x89, 0x9a, 0x9a, 0x08, 0x0b, 0x8e, 0x2e, - 0xc2, 0x79, 0x11, 0xcb, 0x44, 0xbb, 0x90, 0xa0, 0x2a, 0x69, 0x7b, 0x34, 0xfa, 0x8a, 0x30, 0xf0, - 0xc6, 0x61, 0x3f, 0x2f, 0x6f, 0x32, 0xe1, 0x85, 0xbb, 0x90, 0x39, 0x78, 0xdd, 0x44, 0x79, 0x98, - 0xf6, 0x1c, 0x0d, 0x3f, 0xb2, 0x88, 0x67, 0xd9, 0xbb, 0xcc, 0x2f, 0x92, 0x55, 0xf0, 0x9c, 0x9a, - 0x90, 0xe4, 0xfe, 0x3c, 0x02, 0xe8, 0x28, 0x15, 0xa2, 0xbf, 0x93, 0xe0, 0x45, 0xdf, 0x59, 0x72, - 0x5c, 0x6b, 0xd7, 0xb2, 0xf5, 0xf6, 0x90, 0xd7, 0x24, 0xb1, 0x7d, 0xf8, 0xf4, 0x22, 0x7c, 0x2b, - 0x3c, 0xa9, 0x0d, 0x01, 0x3f, 0xea, 0x51, 0xbd, 0x48, 0x0d, 0x3b, 0xf7, 0xa8, 0x8e, 0x54, 0xb9, - 0xa7, 0x66, 0x7b, 0x27, 0x34, 0xce, 0xad, 0xc2, 0x4b, 0xa7, 0x02, 0x9f, 0xc7, 0x4c, 0xe7, 0x7e, - 0x28, 0xc1, 0x95, 0x13, 0x8c, 0x5f, 0x18, 0x27, 0xc5, 0x71, 0xee, 0x86, 0x71, 0xa6, 0x17, 0xbf, - 0x7b, 0x01, 0x03, 0x1b, 0x1e, 0xc4, 0x32, 0xbc, 0x70, 0x22, 0x81, 0x9c, 0x35, 0x1b, 0x39, 0x0c, - 0xf4, 0x2f, 0x12, 0xcc, 0x8c, 0x9c, 0x07, 0xf4, 0x49, 0x48, 0xc1, 0xeb, 0x34, 0xbe, 0x65, 0x9d, - 0x3c, 0x17, 0x2d, 0xdf, 0x3f, 0xaa, 0xe5, 0xeb, 0xb4, 0x07, 0xd6, 0x31, 0xeb, 0xe1, 0x83, 0x67, - 0xee, 0x81, 0x43, 0x0c, 0x34, 0xbd, 0x11, 0x95, 0x25, 0x25, 0x52, 0x78, 0x93, 0x4e, 0x90, 0xad, - 0x6c, 0x10, 0xf0, 0xbc, 0x04, 0xb0, 0x67, 0xed, 0xee, 0x69, 0x0f, 0x75, 0x0f, 0xbb, 0x22, 0xb5, - 0x90, 0xa0, 0x92, 0x8f, 0xa8, 0xa0, 0xf0, 0xa7, 0x00, 0xa9, 0x7a, 0xa7, 0xeb, 0xb8, 0x9e, 0xef, - 0x75, 0xdd, 0x81, 0x18, 0x27, 0x7c, 0xa1, 0xe4, 0xc5, 0x53, 0xb6, 0x71, 0xa8, 0x25, 0xb7, 0xd3, - 0x82, 0x61, 0x04, 0x06, 0xda, 0x80, 0x38, 0x37, 0x8e, 0x24, 0x7b, 0x85, 0xc1, 0x2d, 0x8c, 0x0d, - 0xc7, 0xcd, 0xac, 0x6f, 0x1b, 0x05, 0x4a, 0xe0, 0x1f, 0x45, 0x8e, 0xf5, 0x8f, 0xde, 0x87, 0x18, - 0x4f, 0x4c, 0x89, 0xc8, 0x31, 0x7f, 0x4c, 0xc8, 0x59, 0xdf, 0x58, 0xb2, 0xda, 0x78, 0x89, 0x55, - 0xf3, 0x47, 0xcb, 0x1b, 0xa1, 0x57, 0x40, 0x26, 0xc4, 0xd3, 0x88, 0xf5, 0x03, 0xee, 0x71, 0x4e, - 0xf2, 0x9c, 0x47, 0xb3, 0xb9, 0xd5, 0xb4, 0x7e, 0x80, 0xd5, 0x38, 0x21, 0x1e, 0xfd, 0x81, 0xae, - 0x02, 0xb3, 0xbe, 0x44, 0xa7, 0x36, 0x95, 0x99, 0xcf, 0x49, 0x35, 0x24, 0x61, 0x38, 0xfb, 0x56, - 0x57, 0xdb, 0xd9, 0x27, 0xdc, 0x66, 0x09, 0x9c, 0x7d, 0xab, 0xbb, 0xb4, 0x4a, 0xd4, 0x38, 0x2d, - 0x5c, 0xda, 0x27, 0x28, 0x07, 0xf2, 0x43, 0xbd, 0xdd, 0x66, 0x1e, 0xee, 0x14, 0x43, 0x09, 0xbe, - 0x87, 0x49, 0x32, 0xf6, 0xd5, 0x92, 0xa4, 0xf0, 0x29, 0xbb, 0xba, 0xb7, 0xc7, 0xc2, 0x91, 0x84, - 0x0a, 0x5c, 0xb4, 0xa9, 0x7b, 0x7b, 0x28, 0x0b, 0x71, 0x3e, 0x2f, 0x92, 0x95, 0xe7, 0x27, 0xaf, - 0x27, 0x55, 0xff, 0x13, 0xbd, 0x0a, 0x33, 0x16, 0x0b, 0xb1, 0x35, 0xd3, 0x72, 0xb1, 0xe1, 0xb5, - 0x0f, 0x98, 0xbd, 0x95, 0xd5, 0x34, 0x17, 0x57, 0x85, 0x14, 0xdd, 0x00, 0x65, 0xd4, 0x41, 0x61, - 0x76, 0x52, 0x56, 0x67, 0x46, 0xfc, 0x13, 0xea, 0xcb, 0x88, 0xbd, 0x0e, 0x19, 0xfe, 0x2c, 0xf7, - 0x65, 0x44, 0xc1, 0xc0, 0xe8, 0xdf, 0x00, 0x45, 0x78, 0x27, 0x83, 0xba, 0x29, 0x8e, 0xcb, 0xe5, - 0x83, 0xaa, 0x45, 0x98, 0xed, 0xea, 0x2e, 0xc1, 0x5a, 0xab, 0x67, 0x9b, 0x6d, 0xac, 0x71, 0xac, - 0x6c, 0x9a, 0xd5, 0xce, 0xb0, 0xa2, 0x32, 0x2b, 0xe1, 0x7a, 0x77, 0x56, 0xd4, 0x76, 0xf9, 0xff, - 0x21, 0x6a, 0xcb, 0xfd, 0x22, 0x02, 0x53, 0xec, 0x94, 0xa1, 0x5b, 0x10, 0xa5, 0x7b, 0x29, 0xf2, - 0x0c, 0xe3, 0x7a, 0xce, 0xac, 0x0d, 0x42, 0x10, 0xb5, 0xf5, 0x0e, 0xce, 0x22, 0xb6, 0xd3, 0xec, - 0x37, 0xba, 0x02, 0x71, 0x82, 0xef, 0x6b, 0x0f, 0xf4, 0x76, 0x76, 0x96, 0x29, 0x62, 0x8c, 0xe0, - 0xfb, 0xf7, 0xf4, 0x36, 0xba, 0x04, 0x31, 0x8b, 0x68, 0x36, 0x7e, 0x98, 0x9d, 0xe3, 0x7c, 0x6a, - 0x91, 0x75, 0xfc, 0x10, 0x7d, 0x0d, 0x12, 0x0f, 0x75, 0xa2, 0xe1, 0x4e, 0xd7, 0x3b, 0x60, 0x6b, - 0x21, 0x53, 0xd5, 0x25, 0x35, 0xfa, 0xcd, 0xcc, 0xae, 0xee, 0xee, 0x62, 0x4f, 0x33, 0x9c, 0x36, - 0xc9, 0x5e, 0xa2, 0x47, 0x95, 0x7a, 0xed, 0x54, 0x54, 0x71, 0xda, 0xa4, 0x11, 0x95, 0x23, 0xca, - 0x64, 0x23, 0x2a, 0x4f, 0x2a, 0xd1, 0x46, 0x54, 0x8e, 0x2a, 0x53, 0x8d, 0xa8, 0x3c, 0xa5, 0xc4, - 0x1a, 0x51, 0x39, 0xa6, 0xc4, 0x1b, 0x51, 0x39, 0xae, 0xc8, 0x8d, 0xa8, 0x2c, 0x2b, 0x89, 0x46, - 0x54, 0x4e, 0x28, 0xd0, 0x88, 0xca, 0xa0, 0x4c, 0x37, 0xa2, 0xf2, 0xb4, 0x92, 0x6c, 0x44, 0xe5, - 0xa4, 0x92, 0x6a, 0x44, 0xe5, 0x94, 0x92, 0x6e, 0x44, 0xe5, 0xb4, 0x32, 0xd3, 0x88, 0xca, 0x33, - 0x8a, 0xd2, 0x88, 0xca, 0x8a, 0x92, 0x69, 0x44, 0xe5, 0x8c, 0x82, 0x72, 0x35, 0x91, 0xd8, 0xd2, - 0xd1, 0x77, 0x87, 0xd6, 0x69, 0x6c, 0x8f, 0x9e, 0x35, 0x2a, 0xfc, 0x52, 0x02, 0xa5, 0x89, 0xef, - 0xf7, 0xb0, 0x6d, 0xe0, 0x7b, 0x7a, 0xbb, 0xb2, 0xd7, 0xb3, 0xf7, 0xd1, 0x2b, 0x30, 0x63, 0xd0, - 0x1f, 0x1a, 0xcf, 0xf6, 0xd0, 0x15, 0x93, 0xd8, 0x8a, 0xa5, 0x98, 0xb8, 0x49, 0xa5, 0x74, 0xe1, - 0x5e, 0x02, 0x10, 0xf5, 0x28, 0x9b, 0xf0, 0x4c, 0x70, 0x82, 0x57, 0xa1, 0x14, 0x32, 0x02, 0xe3, - 0x3a, 0x0f, 0x19, 0x65, 0x0d, 0xc1, 0xa8, 0xce, 0x43, 0xb4, 0x00, 0x73, 0x36, 0x7e, 0xe4, 0x69, - 0xa3, 0x95, 0x19, 0x3d, 0xa9, 0x19, 0x5a, 0x56, 0x09, 0x37, 0x28, 0xfc, 0x73, 0x04, 0x66, 0xfc, - 0x41, 0xfb, 0x9c, 0xbe, 0x03, 0x0a, 0xdd, 0x5d, 0xcb, 0xd4, 0x3c, 0x87, 0x23, 0xf9, 0xec, 0xfe, - 0xfe, 0x29, 0x74, 0x3c, 0x82, 0x42, 0xbf, 0xeb, 0xe6, 0x96, 0xc3, 0xba, 0xe3, 0x4e, 0x80, 0x9a, - 0x22, 0x61, 0x59, 0x6e, 0x1b, 0xd2, 0x7e, 0x23, 0x2e, 0x41, 0x15, 0x88, 0x0d, 0xf5, 0xf7, 0xda, - 0x18, 0xfd, 0xf9, 0x4b, 0xad, 0x8a, 0xa6, 0xb9, 0xdf, 0x05, 0x74, 0xb4, 0xef, 0xb0, 0xe9, 0x9f, - 0xe2, 0xa6, 0x7f, 0x63, 0xd8, 0x01, 0x79, 0xf7, 0x7c, 0x73, 0x0b, 0x0d, 0x3b, 0x9c, 0xaa, 0xf8, - 0xc7, 0x08, 0xa4, 0xb9, 0x61, 0x0a, 0x6c, 0x2a, 0xa5, 0x28, 0xca, 0x80, 0x96, 0xbd, 0xab, 0x75, - 0x85, 0x90, 0xcd, 0x2f, 0xa2, 0x2a, 0x7e, 0x41, 0x50, 0xf9, 0x1b, 0x90, 0x72, 0xb1, 0x6e, 0x0e, - 0x2a, 0x46, 0x58, 0xc5, 0x24, 0x15, 0x06, 0x95, 0xae, 0x41, 0x9a, 0xf9, 0x3d, 0x83, 0x5a, 0x93, - 0xac, 0x56, 0x8a, 0x49, 0x83, 0x6a, 0x65, 0x48, 0x91, 0xae, 0x6e, 0x0f, 0x6a, 0x45, 0xd9, 0xa2, - 0x9e, 0x91, 0x58, 0x4d, 0xd2, 0x36, 0x61, 0x87, 0xc0, 0xc5, 0xa4, 0xd7, 0xc1, 0x5a, 0xd7, 0xe1, - 0x91, 0xf7, 0xa4, 0x9a, 0xe0, 0x92, 0x4d, 0x87, 0xa0, 0x6d, 0xa6, 0x2a, 0x6c, 0x2d, 0x34, 0x93, - 0x2f, 0x4e, 0x36, 0xc6, 0x7a, 0xb9, 0x39, 0xfe, 0x72, 0xaa, 0x33, 0x64, 0x58, 0x50, 0xf8, 0x6b, - 0x09, 0xae, 0xd0, 0xf8, 0x9b, 0x9f, 0xb4, 0x0a, 0xbb, 0xc3, 0xf1, 0xb5, 0x53, 0x87, 0x38, 0x8b, - 0xe1, 0x03, 0x47, 0x6c, 0xe5, 0xb0, 0x9f, 0x8f, 0xd1, 0xda, 0x17, 0xb6, 0x72, 0x31, 0x0a, 0x5c, - 0x67, 0x11, 0xa5, 0xe7, 0xea, 0x36, 0xb1, 0x68, 0x6c, 0x45, 0xb7, 0xad, 0x83, 0x3b, 0x2d, 0xec, - 0xf2, 0xcd, 0x48, 0xaa, 0x73, 0x43, 0x85, 0x6b, 0xbc, 0xac, 0x90, 0x83, 0xec, 0xe8, 0x90, 0x83, - 0x74, 0xd5, 0x6f, 0xc1, 0xe5, 0x75, 0xfc, 0xf0, 0xb8, 0xd9, 0x94, 0x21, 0xce, 0x99, 0xce, 0x57, - 0xf9, 0xeb, 0xa3, 0xa4, 0x13, 0xbe, 0xc6, 0x2a, 0xb2, 0x91, 0x6e, 0xb1, 0x06, 0xaa, 0xdf, 0xb0, - 0xf0, 0x29, 0x5c, 0x19, 0x41, 0x0f, 0xb6, 0xef, 0x03, 0x88, 0xd1, 0x38, 0x5d, 0xb8, 0x67, 0xe9, - 0xa3, 0x94, 0x76, 0x14, 0xbd, 0x49, 0xeb, 0xab, 0xa2, 0x59, 0x41, 0x65, 0x79, 0xb6, 0x5e, 0x07, - 0x53, 0x0d, 0xb9, 0x63, 0x11, 0x0f, 0x7d, 0x08, 0x49, 0xa1, 0x11, 0x54, 0x51, 0xfc, 0x61, 0x9f, - 0xa1, 0x54, 0xd3, 0x6e, 0x00, 0x42, 0x0a, 0x7f, 0x23, 0xc1, 0x6c, 0xd5, 0x75, 0xba, 0x5d, 0x6c, - 0x0a, 0x93, 0xc3, 0xd7, 0xc2, 0xb7, 0x34, 0x52, 0xc8, 0xd2, 0xac, 0x43, 0xa4, 0x5e, 0x15, 0xfe, - 0xf0, 0xed, 0x8b, 0xba, 0xd9, 0xf5, 0x2a, 0x7a, 0x97, 0x2f, 0x48, 0x8f, 0x30, 0xfe, 0x4c, 0x1f, - 0x49, 0xb7, 0x0c, 0xa9, 0x29, 0xab, 0xa8, 0x8a, 0x06, 0x85, 0x9f, 0xc7, 0xe1, 0x52, 0x78, 0x91, - 0x97, 0x2b, 0xfe, 0xc0, 0x3f, 0x83, 0xb8, 0x1f, 0x72, 0x8f, 0xc1, 0x93, 0xc7, 0x41, 0x14, 0xc5, - 0x7a, 0x84, 0xc3, 0x6e, 0x1f, 0x13, 0x35, 0x21, 0x63, 0xd9, 0x1e, 0x76, 0xdb, 0x58, 0x7f, 0x40, - 0x9d, 0x0b, 0xba, 0x66, 0x22, 0xc5, 0x39, 0xae, 0x29, 0x57, 0x42, 0x00, 0xdc, 0x25, 0xf8, 0x0c, - 0x66, 0xc3, 0xa0, 0xfe, 0xf8, 0x4f, 0xcf, 0xad, 0xb1, 0xe1, 0x0d, 0x60, 0xfd, 0x24, 0x60, 0x08, - 0x48, 0x24, 0x08, 0xd0, 0xc7, 0x41, 0x5c, 0xc0, 0xf3, 0xa7, 0xb7, 0x9e, 0x79, 0x45, 0xaa, 0x23, - 0x31, 0xc2, 0x90, 0xa7, 0xcb, 0x8c, 0xe0, 0x57, 0xe4, 0xe9, 0xde, 0x83, 0x18, 0x4f, 0xb1, 0x89, - 0x3b, 0x83, 0xdb, 0xcf, 0x3a, 0x05, 0x9e, 0xba, 0x53, 0x05, 0x5a, 0xee, 0x4f, 0x24, 0x48, 0x86, - 0xb7, 0x1b, 0x59, 0x20, 0xb3, 0xe5, 0xf7, 0x29, 0x6d, 0xf2, 0xb9, 0x47, 0x7e, 0x5c, 0x95, 0xea, - 0x26, 0x75, 0xc4, 0x4c, 0xd7, 0xe9, 0x0e, 0xee, 0x8c, 0x26, 0x55, 0x99, 0x0a, 0xa8, 0xeb, 0x98, - 0xfb, 0x3d, 0x48, 0x04, 0x8b, 0x1e, 0xca, 0xe5, 0x4c, 0x3e, 0xc7, 0x5c, 0xce, 0xa9, 0xfd, 0x57, - 0x21, 0x35, 0xb4, 0x62, 0xe8, 0x72, 0x30, 0x86, 0x68, 0x39, 0xc6, 0xc7, 0x70, 0x26, 0x4a, 0xe1, - 0xaf, 0xe2, 0x30, 0x7b, 0x1c, 0xd3, 0x7e, 0x02, 0x4a, 0x88, 0xb7, 0xb4, 0xb6, 0x45, 0x3c, 0xa1, - 0x9b, 0x37, 0x4e, 0x4f, 0x3d, 0x84, 0xc8, 0x4f, 0xa8, 0x62, 0xda, 0x1d, 0xa6, 0xc4, 0xef, 0x41, - 0xda, 0xe4, 0x03, 0x17, 0xd9, 0x4f, 0x71, 0xf3, 0x7b, 0x5a, 0x30, 0x7c, 0x0c, 0x01, 0x0a, 0xf4, - 0x94, 0x19, 0x2a, 0x22, 0xc8, 0x80, 0x54, 0x00, 0x7e, 0xd0, 0x15, 0x51, 0xd5, 0xc5, 0xc9, 0x30, - 0xe9, 0xf7, 0x42, 0x31, 0xd1, 0x2e, 0xcc, 0xf8, 0x9d, 0xf8, 0x01, 0x78, 0xe2, 0xb9, 0x74, 0xe3, - 0x2f, 0x4c, 0x53, 0x04, 0xe4, 0x3f, 0x92, 0x60, 0xd6, 0xef, 0x29, 0xb8, 0x25, 0xb0, 0x4c, 0x46, - 0x67, 0xa9, 0x72, 0xf3, 0xb0, 0x9f, 0xcf, 0x88, 0x95, 0xf1, 0xaf, 0x05, 0x2e, 0xac, 0x77, 0x19, - 0x73, 0x04, 0xd0, 0xa4, 0x3e, 0x04, 0x2d, 0xa7, 0x1d, 0x4f, 0x0d, 0x7c, 0x08, 0x4a, 0x6c, 0x17, - 0xf7, 0x21, 0xe8, 0xcf, 0xba, 0x89, 0xfe, 0x50, 0x82, 0x0c, 0xbf, 0xb4, 0xe8, 0xf4, 0x3c, 0x9d, - 0xdf, 0x0f, 0xfa, 0x91, 0xf9, 0x27, 0x87, 0xfd, 0xfc, 0x0c, 0xdb, 0xde, 0x35, 0x51, 0xc6, 0xba, - 0x2d, 0x3f, 0x6b, 0xb7, 0x03, 0x14, 0x11, 0xc8, 0x06, 0x02, 0x13, 0xad, 0x42, 0x9a, 0xa7, 0x2b, - 0xfc, 0xa7, 0x2d, 0x2c, 0x64, 0x4f, 0x95, 0x5f, 0x7e, 0xda, 0xcf, 0xcf, 0x1f, 0x73, 0x4e, 0x78, - 0xa6, 0xe3, 0x1e, 0xaf, 0xab, 0xa6, 0x76, 0xc2, 0x9f, 0xe8, 0x0e, 0xcc, 0x70, 0xc7, 0x33, 0x08, - 0x70, 0x59, 0xc2, 0x62, 0xcc, 0xab, 0x36, 0xee, 0xb4, 0x06, 0x52, 0x91, 0x85, 0xba, 0x0c, 0x73, - 0xc7, 0xfa, 0x4c, 0x5f, 0xc4, 0xe0, 0xf2, 0x30, 0xad, 0x06, 0x5e, 0x8d, 0x36, 0x6a, 0x6f, 0x3f, - 0x18, 0x9b, 0x9a, 0x7d, 0x0c, 0x4e, 0x8d, 0xfe, 0xd7, 0xa8, 0xc5, 0xfd, 0x6c, 0xc4, 0x7a, 0x3d, - 0x03, 0x3e, 0xdb, 0xde, 0x11, 0x7c, 0xdf, 0x84, 0x7d, 0x1c, 0x58, 0x16, 0x9e, 0x77, 0xfa, 0xf0, - 0x19, 0xe0, 0x59, 0x7b, 0xff, 0x33, 0xb0, 0x2d, 0xff, 0x24, 0x41, 0x6a, 0x68, 0x66, 0xbf, 0x49, - 0xe3, 0xb2, 0x19, 0xf8, 0x56, 0xfc, 0x01, 0xd2, 0x3b, 0xe7, 0x9f, 0xd6, 0xb0, 0xcb, 0x95, 0xfb, - 0x5b, 0x09, 0x52, 0x43, 0x0b, 0xf9, 0x15, 0x99, 0xa5, 0xe7, 0x3f, 0xf2, 0x16, 0xa4, 0x87, 0xb7, - 0x28, 0xd4, 0x87, 0xf4, 0x7c, 0xfa, 0x28, 0x7c, 0x07, 0x62, 0x5c, 0x82, 0x10, 0xa4, 0x3f, 0x2a, - 0xd5, 0xb7, 0xea, 0xeb, 0xcb, 0xda, 0xd2, 0x86, 0xaa, 0x2d, 0x57, 0x94, 0x09, 0x94, 0x04, 0xb9, - 0x5a, 0xbb, 0x53, 0xa3, 0x42, 0x45, 0x42, 0xd3, 0x10, 0x67, 0x5f, 0xb5, 0xaa, 0x12, 0x29, 0x94, - 0x41, 0xe1, 0xd8, 0x3b, 0x98, 0x9a, 0x19, 0x1a, 0x45, 0xa0, 0x22, 0xcc, 0x32, 0x97, 0xbf, 0x43, - 0x3d, 0x2b, 0x7a, 0xbc, 0xb5, 0x90, 0x2f, 0x9e, 0x09, 0x8a, 0xe8, 0xe9, 0x5d, 0xd7, 0x3b, 0xb8, - 0xf0, 0xab, 0x28, 0x64, 0x06, 0x20, 0xbe, 0x91, 0xfd, 0x4b, 0x69, 0x10, 0xcf, 0xc4, 0xce, 0xbc, - 0x7d, 0x3a, 0xd2, 0x5e, 0x84, 0x36, 0xe2, 0xc2, 0xfc, 0x23, 0x7a, 0x68, 0x9e, 0xf6, 0xf3, 0x99, - 0xd1, 0xc1, 0x92, 0x0b, 0xde, 0xa4, 0xfb, 0x43, 0x64, 0x99, 0x57, 0xcb, 0xde, 0xd7, 0x06, 0xcf, - 0x60, 0x78, 0xe6, 0xd5, 0xb2, 0xf7, 0xb7, 0xd5, 0xba, 0x1a, 0xa7, 0x85, 0xdb, 0xae, 0x85, 0x1a, - 0x10, 0x75, 0xba, 0x9e, 0x1f, 0x40, 0xbf, 0x7d, 0xae, 0x29, 0x6d, 0x74, 0xc5, 0x7c, 0x54, 0x86, - 0x81, 0x1a, 0xfc, 0x42, 0x75, 0xb0, 0xd0, 0x8c, 0x90, 0xc7, 0xa4, 0xd0, 0xd4, 0xd0, 0x46, 0xe4, - 0x76, 0x21, 0x19, 0x5e, 0xb1, 0x63, 0x6e, 0x59, 0x4a, 0xc3, 0x49, 0x8e, 0xd7, 0xc6, 0x1a, 0xba, - 0x08, 0x30, 0x43, 0x97, 0x21, 0xdf, 0x81, 0x44, 0x30, 0x8f, 0xf3, 0xdc, 0x09, 0x71, 0x8e, 0x0f, - 0x32, 0x78, 0x53, 0x4a, 0xac, 0xd0, 0x8f, 0x40, 0x52, 0xc5, 0xc4, 0x69, 0x3f, 0xc0, 0x26, 0xf5, - 0xa0, 0x82, 0x67, 0x5f, 0xd2, 0xf8, 0xcf, 0xbe, 0x4a, 0x90, 0x18, 0x58, 0xa0, 0x73, 0x3c, 0x7d, - 0x1a, 0xb4, 0x42, 0xb7, 0xe1, 0x6b, 0xe1, 0xb7, 0x45, 0x4e, 0xcf, 0x36, 0x75, 0xf7, 0x40, 0x73, - 0xb1, 0x6e, 0xec, 0x61, 0x53, 0x5c, 0xfe, 0xbd, 0x10, 0x7a, 0x5c, 0x24, 0x6a, 0xa8, 0xbc, 0x02, - 0xfa, 0x04, 0x52, 0x41, 0x23, 0xea, 0x97, 0x31, 0x0f, 0x26, 0xbd, 0xf8, 0xad, 0xd3, 0x7d, 0xc9, - 0x60, 0xd6, 0x45, 0x1f, 0x8f, 0xfa, 0x5f, 0x6a, 0xb2, 0x15, 0xfa, 0x2a, 0xbc, 0x0f, 0xc9, 0x70, - 0x29, 0x92, 0x21, 0xba, 0xbe, 0xb1, 0x5e, 0xe3, 0x67, 0xba, 0x5c, 0xaa, 0xac, 0x2e, 0xd5, 0xef, - 0xdc, 0x51, 0x24, 0x2a, 0xaf, 0x7d, 0x5c, 0xdf, 0x52, 0x22, 0xf4, 0x74, 0xab, 0xb5, 0xe6, 0x56, - 0x49, 0xdd, 0x52, 0x26, 0x0b, 0x18, 0x52, 0xe1, 0x9e, 0x28, 0x67, 0x52, 0x87, 0x95, 0x09, 0x86, - 0x62, 0xf6, 0x57, 0xc7, 0x1c, 0xab, 0xaf, 0x7b, 0x6e, 0x18, 0xb5, 0xf0, 0xe3, 0x08, 0xa0, 0x81, - 0xca, 0x84, 0x08, 0x7a, 0xb4, 0xb3, 0xc8, 0xc5, 0x3b, 0x43, 0x3f, 0x39, 0x3d, 0xbd, 0x3e, 0xc9, - 0xd2, 0xeb, 0x8c, 0x2f, 0x7e, 0xa3, 0x29, 0x76, 0xe1, 0xc2, 0xfc, 0x77, 0x14, 0x50, 0xc5, 0xc5, - 0xba, 0x87, 0x29, 0x1f, 0x93, 0xd3, 0xf2, 0x19, 0x65, 0x98, 0xe2, 0xf1, 0x7b, 0xe4, 0x3c, 0xf1, - 0xbb, 0x58, 0x14, 0xde, 0x14, 0x7d, 0x1f, 0x92, 0x86, 0xd3, 0xee, 0x75, 0x6c, 0x8d, 0xbd, 0xc1, - 0x10, 0xc1, 0xc6, 0xb7, 0x4f, 0x3b, 0xda, 0x47, 0x06, 0x57, 0xac, 0x38, 0x6d, 0xfa, 0x1d, 0xbc, - 0x56, 0x64, 0x80, 0xac, 0x06, 0x7a, 0x11, 0x12, 0x01, 0xcd, 0x30, 0xb5, 0x4e, 0xa8, 0x03, 0x01, - 0x5a, 0x84, 0x29, 0x9d, 0x68, 0xce, 0x0e, 0xf3, 0x9c, 0xcf, 0x3a, 0x77, 0x6a, 0x54, 0x27, 0x1b, - 0x3b, 0xe8, 0x26, 0x64, 0x3a, 0xfa, 0x23, 0x6d, 0xc7, 0xd5, 0x0d, 0xe1, 0x09, 0xb7, 0x39, 0xed, - 0x49, 0xea, 0x4c, 0x47, 0x7f, 0xb4, 0x24, 0xe4, 0x75, 0xb3, 0x8d, 0xd1, 0x5b, 0x90, 0xda, 0xb9, - 0xcf, 0xe3, 0x28, 0x6e, 0x82, 0xf8, 0x83, 0x96, 0x99, 0xc3, 0x7e, 0x7e, 0x7a, 0xe9, 0x2e, 0x5b, - 0x18, 0x6a, 0x80, 0xd4, 0xe9, 0x9d, 0xfb, 0xc1, 0x47, 0xee, 0xbf, 0x24, 0x88, 0x8b, 0x19, 0xa1, - 0x2e, 0x80, 0x58, 0x1e, 0xcb, 0xe4, 0xaa, 0x9e, 0x2a, 0xdf, 0x3d, 0xec, 0xe7, 0x13, 0x15, 0x26, - 0xad, 0x57, 0xc9, 0xd3, 0x7e, 0xfe, 0xc3, 0x67, 0x35, 0x1f, 0x3e, 0x88, 0x9a, 0xe0, 0x9d, 0xd4, - 0x4d, 0x96, 0xb4, 0xdd, 0xd3, 0x89, 0xb6, 0x67, 0x11, 0xcf, 0xd9, 0x75, 0xf5, 0x8e, 0xb8, 0x4c, - 0x4e, 0xee, 0xe9, 0x64, 0xc5, 0x97, 0xa1, 0x1c, 0x75, 0xc4, 0x1e, 0xf0, 0x27, 0x34, 0x9c, 0x5d, - 0x82, 0x6f, 0xb4, 0x08, 0x97, 0x82, 0xc6, 0x1a, 0x5d, 0xa9, 0x56, 0xcf, 0xd8, 0xc7, 0xcc, 0xe0, - 0x50, 0x26, 0x9f, 0x0d, 0x0a, 0xd7, 0xf4, 0x47, 0x65, 0x5e, 0x54, 0xb8, 0x04, 0xb3, 0xa1, 0x6d, - 0x0d, 0xdc, 0x66, 0x0c, 0x0a, 0xbf, 0xf3, 0x0e, 0xbd, 0xbf, 0xbd, 0x0b, 0x33, 0x23, 0xef, 0xdb, - 0x05, 0xd9, 0x86, 0x93, 0x8d, 0xc3, 0x0f, 0xe2, 0x8b, 0x15, 0xfe, 0xe9, 0x07, 0x02, 0x69, 0x63, - 0xe8, 0xbb, 0x30, 0x0b, 0x99, 0xa0, 0x9b, 0xa0, 0xef, 0x9f, 0xa6, 0x20, 0xbe, 0xa9, 0x1f, 0xb4, - 0x1d, 0xdd, 0x44, 0xf3, 0x30, 0xed, 0xbf, 0x7d, 0xf1, 0xfb, 0x4b, 0xa8, 0x61, 0xd1, 0xb0, 0x9a, - 0x29, 0xec, 0xd6, 0x27, 0xa4, 0x66, 0x16, 0xa4, 0x7b, 0x04, 0xbb, 0x54, 0x03, 0x34, 0xf6, 0x18, - 0x9f, 0xdb, 0x96, 0x72, 0xf9, 0x69, 0x3f, 0x7f, 0x7b, 0xbc, 0xcd, 0xc3, 0x46, 0xcf, 0xb5, 0xbc, - 0x83, 0x62, 0xf3, 0xee, 0x9d, 0x6d, 0x01, 0x45, 0x8f, 0xb8, 0xa3, 0xa6, 0x7a, 0xe1, 0x4f, 0xf1, - 0xc4, 0x89, 0x6e, 0x84, 0xd6, 0xb1, 0x0c, 0xd7, 0x21, 0xfe, 0xdd, 0x8a, 0x90, 0xae, 0x31, 0x21, - 0x7a, 0x15, 0x66, 0x76, 0x2c, 0x9b, 0x5d, 0x0f, 0xfa, 0xf5, 0xf8, 0xb5, 0x4a, 0xda, 0x17, 0x8b, - 0x8a, 0x0f, 0x20, 0x1d, 0x7a, 0x5b, 0x44, 0x95, 0x30, 0xc6, 0x94, 0x70, 0xe3, 0xb0, 0x9f, 0x4f, - 0x0d, 0x0e, 0x35, 0x57, 0xc4, 0x8b, 0xf8, 0x31, 0xa9, 0x41, 0x37, 0x54, 0x0d, 0xe7, 0x60, 0x8a, - 0xfd, 0xa9, 0x06, 0x7f, 0x9c, 0xa9, 0xf2, 0x0f, 0x54, 0x83, 0x94, 0xc8, 0x7b, 0xf0, 0xbf, 0xe3, - 0x10, 0x0f, 0xb2, 0xe6, 0x43, 0x5b, 0xef, 0xff, 0xa5, 0x47, 0xb1, 0x66, 0x1b, 0x8e, 0x89, 0xcd, - 0x1a, 0xfd, 0x56, 0x45, 0x9a, 0x97, 0x7d, 0x10, 0xb4, 0x0c, 0x69, 0xa3, 0x8d, 0x75, 0xbb, 0xd7, - 0xf5, 0x71, 0xd0, 0x98, 0x38, 0x29, 0xd1, 0x4e, 0x00, 0xad, 0x03, 0xda, 0x61, 0x2f, 0x64, 0xc2, - 0xa3, 0x62, 0xd7, 0x88, 0xe3, 0x80, 0x29, 0xac, 0xad, 0x3a, 0x18, 0x19, 0x7a, 0x1b, 0xa6, 0xda, - 0x58, 0x27, 0x58, 0xbc, 0xbc, 0x9d, 0x3f, 0x85, 0x06, 0xd9, 0xdf, 0x72, 0xa8, 0xbc, 0x3a, 0x7a, - 0x19, 0x52, 0xb6, 0x63, 0x1b, 0xba, 0x6d, 0xe0, 0x36, 0x63, 0x64, 0x7e, 0x63, 0x39, 0x2c, 0x44, - 0x65, 0x88, 0xf1, 0xbb, 0x6d, 0x11, 0xe8, 0x5e, 0x1f, 0xf7, 0xe9, 0xf3, 0xca, 0x84, 0x2a, 0x5a, - 0xa2, 0x1a, 0xc4, 0x5d, 0xfe, 0xce, 0x82, 0xdd, 0x77, 0x9f, 0x99, 0x70, 0x0a, 0xbd, 0x75, 0x59, - 0x99, 0x50, 0xfd, 0xb6, 0x68, 0xcb, 0x7f, 0x76, 0xc8, 0xad, 0xae, 0x78, 0x39, 0x56, 0x1c, 0x33, - 0x8c, 0x18, 0x00, 0x0e, 0xa1, 0xd0, 0x09, 0x5a, 0xec, 0xbe, 0x8a, 0xdd, 0x84, 0x9f, 0x3e, 0xc1, - 0xa1, 0x17, 0x17, 0x74, 0x82, 0xbc, 0x25, 0x5a, 0x07, 0x30, 0x02, 0x4f, 0x80, 0xdd, 0x91, 0x4f, - 0x2f, 0xbe, 0x7e, 0x1e, 0x27, 0x79, 0x65, 0x42, 0x0d, 0x21, 0xa0, 0xbb, 0x30, 0x6d, 0x0c, 0xa8, - 0x2d, 0x3b, 0xc3, 0x00, 0xdf, 0x38, 0x97, 0x7d, 0x5b, 0xa1, 0x36, 0x6d, 0x20, 0x45, 0x9f, 0x42, - 0x9a, 0x0c, 0x05, 0x55, 0xd9, 0x4b, 0x0c, 0xf5, 0xcd, 0xf3, 0x26, 0x75, 0x57, 0x26, 0xd4, 0x11, - 0x24, 0xf4, 0xdb, 0xa0, 0x78, 0x23, 0x37, 0x3f, 0xec, 0x92, 0xfb, 0xf4, 0x67, 0x7d, 0x27, 0xdc, - 0x6f, 0xad, 0x4c, 0xa8, 0x47, 0xd0, 0xd0, 0x67, 0x30, 0x43, 0x86, 0xff, 0x44, 0x21, 0x7b, 0x85, - 0x75, 0xf0, 0xcd, 0x53, 0xaf, 0x2f, 0x8e, 0xfb, 0x63, 0x8c, 0x95, 0x09, 0x75, 0x14, 0x8b, 0xc2, - 0xdb, 0xc3, 0x17, 0x48, 0xec, 0x09, 0xc5, 0xe9, 0xf0, 0xc7, 0x5f, 0x68, 0x51, 0xf8, 0x11, 0x2c, - 0xb4, 0x0a, 0x89, 0x8e, 0x6f, 0x2b, 0xb2, 0x2f, 0x9c, 0x19, 0x87, 0x8c, 0x9a, 0xaf, 0x95, 0x09, - 0x75, 0xd0, 0x9e, 0x86, 0x6c, 0x9e, 0xab, 0x1b, 0x2c, 0x69, 0x98, 0x63, 0x99, 0x61, 0x16, 0xb2, - 0x6d, 0x51, 0x19, 0x0b, 0xed, 0xd8, 0x0f, 0xb3, 0x9c, 0x80, 0xb8, 0xb8, 0x90, 0x0c, 0x5e, 0x0b, - 0xc4, 0x15, 0xb9, 0xf0, 0xbf, 0x32, 0xc8, 0x81, 0xaf, 0xba, 0x00, 0x28, 0x70, 0x49, 0x06, 0x2f, - 0x61, 0xa9, 0xa9, 0x8a, 0xac, 0x4c, 0xa8, 0x19, 0xbf, 0x6c, 0xf0, 0x18, 0xf6, 0xf6, 0xd0, 0xf3, - 0xa8, 0x71, 0xfe, 0xdc, 0x84, 0x0e, 0x3e, 0x78, 0x3f, 0x45, 0x4d, 0x48, 0xc7, 0x31, 0xad, 0x1d, - 0x6b, 0x60, 0x42, 0x78, 0xfa, 0x3a, 0xed, 0x8b, 0x85, 0x09, 0xb9, 0x06, 0x69, 0xb7, 0x67, 0xb3, - 0xbb, 0x47, 0x91, 0x34, 0xe0, 0x7e, 0x58, 0x4a, 0x48, 0x45, 0xdc, 0x5f, 0x19, 0x61, 0xa7, 0x1b, - 0x67, 0xb2, 0x93, 0x3f, 0xf7, 0x15, 0x29, 0xa0, 0xa7, 0xa5, 0x51, 0x7a, 0xba, 0x79, 0x36, 0x3d, - 0x85, 0x60, 0x02, 0x7e, 0xda, 0x3e, 0x96, 0x9f, 0x16, 0xc6, 0x3c, 0x60, 0x21, 0xc4, 0x61, 0x82, - 0xaa, 0x8c, 0x10, 0xd4, 0x8d, 0x33, 0x09, 0x2a, 0x3c, 0x47, 0xc1, 0x50, 0x1b, 0xc7, 0x30, 0xd4, - 0x1b, 0x63, 0x31, 0x54, 0x08, 0x2c, 0x4c, 0x51, 0xea, 0x71, 0x14, 0x55, 0x1c, 0x8f, 0xa2, 0x42, - 0x90, 0x43, 0x1c, 0xf5, 0xbd, 0x23, 0x1c, 0xa5, 0x9c, 0x7d, 0xc8, 0x8f, 0xcd, 0x14, 0xad, 0x48, - 0x47, 0x48, 0x4a, 0x3f, 0x86, 0xa4, 0x32, 0x0c, 0xfe, 0xad, 0x73, 0x90, 0x54, 0xa8, 0x83, 0xa3, - 0x2c, 0xf5, 0x31, 0x24, 0xc3, 0xcc, 0xc2, 0x5e, 0x0c, 0x9d, 0xce, 0x81, 0x27, 0xfc, 0xdd, 0x15, - 0xd3, 0x81, 0x50, 0x11, 0xfa, 0xfe, 0x51, 0x82, 0x9a, 0x3d, 0x13, 0xfc, 0x84, 0x3b, 0xf1, 0x15, - 0xe9, 0x28, 0x43, 0xdd, 0x09, 0x33, 0xd4, 0xdc, 0x99, 0xf6, 0xeb, 0x88, 0xe7, 0xbb, 0x22, 0x85, - 0x28, 0xaa, 0x0c, 0x20, 0xfb, 0x4f, 0x2e, 0x42, 0x34, 0x54, 0xf8, 0x99, 0x04, 0x93, 0x0d, 0xa7, - 0x85, 0x5e, 0x0a, 0xa5, 0x30, 0x53, 0x22, 0x64, 0x9d, 0x6a, 0x38, 0x2d, 0x91, 0x8b, 0xfc, 0x60, - 0xd0, 0x5a, 0x04, 0x89, 0xdf, 0x38, 0x65, 0x28, 0x41, 0x06, 0x38, 0x68, 0x84, 0xde, 0x83, 0x78, - 0x97, 0x3b, 0xe1, 0x82, 0xa1, 0x0a, 0xa7, 0xb5, 0xe7, 0x35, 0x55, 0xbf, 0xc9, 0xcd, 0x1b, 0xe1, - 0x3f, 0xa4, 0x5c, 0x73, 0x4c, 0x8c, 0xd2, 0x00, 0x9b, 0x3a, 0x21, 0xdd, 0x3d, 0x57, 0x27, 0x58, - 0x99, 0x40, 0x71, 0x98, 0x5c, 0x5d, 0x6b, 0x2a, 0xd2, 0xcd, 0x8f, 0xc3, 0xf9, 0xc7, 0xaa, 0x5a, - 0xaa, 0xaf, 0xd7, 0xd7, 0x97, 0xb5, 0xf5, 0xd2, 0x5a, 0xad, 0xa9, 0x4c, 0xa0, 0x2c, 0xcc, 0x7d, - 0x54, 0xaa, 0x6f, 0x89, 0x84, 0xa4, 0x56, 0x5f, 0xdf, 0xaa, 0xa9, 0xf7, 0x4a, 0x77, 0x14, 0x09, - 0x5d, 0x06, 0xa4, 0x6e, 0x54, 0x56, 0x9b, 0xd5, 0xb2, 0x56, 0xd9, 0x58, 0xdb, 0x2c, 0x55, 0xb6, - 0xea, 0x1b, 0xeb, 0x4a, 0x04, 0xc9, 0x10, 0xad, 0x6e, 0xac, 0xd7, 0x14, 0xb8, 0xf9, 0x93, 0x28, - 0x44, 0x59, 0xea, 0xe3, 0x65, 0x98, 0xde, 0x5e, 0x6f, 0x6e, 0xd6, 0x2a, 0xf5, 0xa5, 0x7a, 0xad, - 0xaa, 0x4c, 0xe4, 0x66, 0x1f, 0x3f, 0x99, 0x9f, 0xa1, 0x45, 0xdb, 0x36, 0xe9, 0x62, 0x83, 0x71, - 0x26, 0xca, 0x41, 0xac, 0x5c, 0xaa, 0xac, 0x6e, 0x6f, 0x2a, 0x52, 0x2e, 0xfd, 0xf8, 0xc9, 0x3c, - 0xd0, 0x0a, 0x9c, 0xef, 0xd0, 0x8b, 0x3c, 0x35, 0xb2, 0xa1, 0xd6, 0x94, 0x48, 0x6e, 0xe6, 0xf1, - 0x93, 0xf9, 0x69, 0x96, 0x71, 0x11, 0x9c, 0xf5, 0x2a, 0xa4, 0x9a, 0x95, 0x95, 0xda, 0x5a, 0x49, - 0xab, 0xac, 0x94, 0xd6, 0x97, 0x6b, 0xca, 0x64, 0x6e, 0xee, 0xf1, 0x93, 0x79, 0x65, 0x54, 0xef, - 0x69, 0x17, 0xf5, 0xb5, 0xcd, 0x0d, 0x75, 0x4b, 0x89, 0x0e, 0xba, 0xe0, 0x74, 0x83, 0x0a, 0x00, - 0xbc, 0xf5, 0x52, 0xad, 0x56, 0x55, 0xa6, 0x72, 0xe8, 0xf1, 0x93, 0xf9, 0x34, 0x2d, 0x1f, 0xb0, - 0x08, 0xba, 0x06, 0xc9, 0x8a, 0x5a, 0x2b, 0x6d, 0xd5, 0xb4, 0xe6, 0x56, 0x69, 0xab, 0xa9, 0xc4, - 0x06, 0x33, 0x09, 0x31, 0x03, 0x2a, 0x42, 0xa6, 0xb4, 0xbd, 0xb5, 0xa1, 0x0d, 0xd5, 0x8d, 0xe7, - 0xae, 0x3c, 0x7e, 0x32, 0x3f, 0x4b, 0xeb, 0x96, 0x7a, 0x9e, 0x13, 0xae, 0xff, 0x3a, 0x28, 0x43, - 0xe3, 0xd7, 0x96, 0x2b, 0x8a, 0x9c, 0xbb, 0xfc, 0xf8, 0xc9, 0x3c, 0x1a, 0x9d, 0xc2, 0x72, 0x05, - 0x7d, 0x0b, 0x2e, 0x6f, 0x7d, 0xb2, 0x59, 0xab, 0xd6, 0x9a, 0x15, 0x6d, 0x78, 0xda, 0x89, 0x5c, - 0xf6, 0xf1, 0x93, 0xf9, 0x39, 0xda, 0xe6, 0xc8, 0xd4, 0xdf, 0x00, 0xa5, 0xb9, 0xa5, 0xd6, 0x4a, - 0x6b, 0x5a, 0x7d, 0x7d, 0xb9, 0xd6, 0x64, 0x9b, 0x05, 0x83, 0x21, 0x8d, 0x9c, 0x61, 0x3a, 0x85, - 0xf5, 0xda, 0x47, 0x23, 0xf8, 0xd3, 0x83, 0xfa, 0x23, 0xc7, 0x12, 0xcd, 0x43, 0x62, 0xad, 0xbe, - 0xac, 0x96, 0x18, 0x6e, 0x32, 0x97, 0x79, 0xfc, 0x64, 0x3e, 0x45, 0xeb, 0x05, 0x87, 0x2c, 0x27, - 0xff, 0xf8, 0x67, 0x57, 0x27, 0xfe, 0xe2, 0xe7, 0x57, 0x27, 0xca, 0xd7, 0x3f, 0xff, 0x8f, 0xab, - 0x13, 0x9f, 0x1f, 0x5e, 0x95, 0x7e, 0x7d, 0x78, 0x55, 0xfa, 0xe2, 0xf0, 0xaa, 0xf4, 0xef, 0x87, - 0x57, 0xa5, 0x3f, 0xfa, 0xf2, 0xea, 0xc4, 0xaf, 0xbf, 0xbc, 0x3a, 0xf1, 0xc5, 0x97, 0x57, 0x27, - 0x3e, 0x8d, 0x71, 0xbd, 0x6e, 0xc5, 0x58, 0xcc, 0xf8, 0xd6, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, - 0xa9, 0xb6, 0xc9, 0xf0, 0xf9, 0x3e, 0x00, 0x00, + // 5102 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x5b, 0x4b, 0x70, 0x23, 0xc7, + 0x79, 0xe6, 0x80, 0x20, 0x1e, 0x3f, 0x1e, 0x1c, 0x34, 0xb9, 0xbb, 0x10, 0x2c, 0x2d, 0x68, 0x58, + 0x2b, 0xed, 0xae, 0x24, 0x50, 0xa6, 0x6c, 0x59, 0x5a, 0x4b, 0x2b, 0xe1, 0x45, 0x12, 0xe0, 0xf2, + 0xb1, 0x03, 0x72, 0xf5, 0x70, 0xe4, 0xc9, 0x60, 0xa6, 0x49, 0x4e, 0x08, 0xcc, 0x60, 0xa7, 0x07, + 0xbb, 0x4b, 0xa7, 0x2a, 0x49, 0xd9, 0x95, 0x2a, 0xd7, 0x9e, 0x92, 0x83, 0x73, 0x48, 0xb2, 0x55, + 0xae, 0xb2, 0x5d, 0x95, 0x43, 0x72, 0x88, 0x2b, 0xe5, 0xe4, 0x90, 0x53, 0x72, 0xd1, 0x21, 0xa9, + 0xf2, 0x51, 0xc9, 0x01, 0x49, 0xa8, 0x4b, 0x8e, 0xa9, 0xe4, 0xb6, 0x87, 0x54, 0xaa, 0x1f, 0x33, + 0x18, 0x00, 0x7c, 0x80, 0xcb, 0x95, 0x73, 0x21, 0xa7, 0xff, 0xee, 0xfe, 0xfa, 0xf5, 0xf7, 0xf7, + 0xff, 0xfd, 0x77, 0x03, 0x2e, 0xff, 0x8e, 0xdd, 0x22, 0x8b, 0xf4, 0x4f, 0xb7, 0xc5, 0xfe, 0x15, + 0xbb, 0x8e, 0xed, 0xda, 0xe8, 0x05, 0xdd, 0xd6, 0x0f, 0x1c, 0x5b, 0xd3, 0xf7, 0x8b, 0xe4, 0x7e, + 0xbb, 0xc8, 0x72, 0x78, 0xa9, 0xdc, 0x25, 0xec, 0x38, 0xb6, 0x43, 0xcb, 0xf3, 0x0f, 0x5e, 0x23, + 0x37, 0xbf, 0x67, 0xef, 0xd9, 0xec, 0x73, 0x91, 0x7e, 0x09, 0x29, 0x62, 0x18, 0xdd, 0xd6, 0xa2, + 0xa1, 0xb9, 0x9a, 0x90, 0x65, 0x3d, 0x99, 0x69, 0xbf, 0xb1, 0x6b, 0x3b, 0x1d, 0xcd, 0xf5, 0x30, + 0xbe, 0x41, 0xee, 0xb7, 0x17, 0x75, 0xcd, 0xd5, 0xda, 0xf6, 0xde, 0xa2, 0x81, 0x89, 0xde, 0x6d, + 0x2d, 0x12, 0xd7, 0xe9, 0xe9, 0x6e, 0xcf, 0xc1, 0x86, 0x28, 0x94, 0x3f, 0xa6, 0x90, 0x8b, 0x2d, + 0xcd, 0x72, 0x3d, 0xfc, 0x9e, 0x6b, 0xb6, 0x17, 0xf7, 0xdb, 0xfa, 0xa2, 0x6b, 0x76, 0x30, 0x71, + 0xb5, 0x4e, 0x57, 0xe4, 0x7c, 0x9d, 0x56, 0x25, 0xfa, 0x3e, 0xee, 0x68, 0xfa, 0xbe, 0x66, 0xed, + 0x61, 0x67, 0x91, 0xb7, 0xa1, 0x77, 0x5b, 0xa2, 0xc8, 0xcb, 0x7a, 0xbb, 0x47, 0x5c, 0xec, 0x3c, + 0xc0, 0x0e, 0x31, 0x6d, 0x6b, 0x51, 0x24, 0x55, 0x91, 0xe6, 0xa5, 0x0a, 0xbf, 0x0f, 0x33, 0x77, + 0xb0, 0x46, 0x30, 0xfa, 0x14, 0xa2, 0x96, 0x6d, 0x60, 0xd5, 0x34, 0xb2, 0xd2, 0x82, 0x74, 0x3d, + 0x55, 0x2e, 0x1d, 0xf5, 0xf3, 0x91, 0x0d, 0xdb, 0xc0, 0xf5, 0xea, 0xd3, 0x7e, 0xfe, 0xad, 0x3d, + 0xd3, 0xdd, 0xef, 0xb5, 0x8a, 0xba, 0xdd, 0x59, 0xf4, 0x67, 0xd4, 0x68, 0x0d, 0xbe, 0x17, 0xbb, + 0x07, 0x7b, 0x8b, 0x62, 0x3e, 0x8a, 0xbc, 0x9a, 0x12, 0xa1, 0x88, 0x75, 0x03, 0xcd, 0xc3, 0x0c, + 0xee, 0xda, 0xfa, 0x7e, 0x36, 0xb4, 0x20, 0x5d, 0x9f, 0x56, 0x78, 0xe2, 0x56, 0xf8, 0x3f, 0x7f, + 0x9a, 0x97, 0x0a, 0xbf, 0x08, 0xc1, 0x95, 0xb2, 0xa6, 0x1f, 0xf4, 0xba, 0x35, 0x4b, 0x77, 0x0e, + 0xbb, 0xae, 0x69, 0x5b, 0x9b, 0xec, 0x2f, 0x41, 0x32, 0x4c, 0x1f, 0xe0, 0x43, 0xd6, 0x9f, 0xa4, + 0x42, 0x3f, 0xd1, 0xfb, 0x10, 0xee, 0xd8, 0x06, 0x66, 0x40, 0xe9, 0xa5, 0x1b, 0xc5, 0x13, 0x17, + 0xb7, 0x38, 0x40, 0x5b, 0xb7, 0x0d, 0xac, 0xb0, 0x6a, 0xa8, 0x05, 0xb1, 0x83, 0x0e, 0x51, 0x4d, + 0x6b, 0xd7, 0xce, 0x4e, 0x2f, 0x48, 0xd7, 0x13, 0x4b, 0xb7, 0x4e, 0x81, 0x38, 0xa1, 0x5b, 0xc5, + 0xb5, 0xf5, 0x66, 0xdd, 0xda, 0xb5, 0xcb, 0x89, 0xa3, 0x7e, 0x3e, 0x2a, 0x12, 0x4a, 0xf4, 0xa0, + 0x43, 0xe8, 0x47, 0x6e, 0x13, 0x3c, 0x19, 0xed, 0x7f, 0xcf, 0x31, 0x59, 0xff, 0xe3, 0x0a, 0xfd, + 0x44, 0xaf, 0x03, 0xc2, 0x1c, 0x0f, 0x1b, 0x2a, 0xd5, 0x24, 0x95, 0x0e, 0x30, 0xc4, 0x06, 0x28, + 0xfb, 0x39, 0x55, 0xcd, 0xd5, 0xd6, 0xf0, 0x21, 0x9f, 0x21, 0x31, 0x4f, 0x7f, 0x30, 0x0d, 0xe9, + 0x41, 0x57, 0x18, 0xfc, 0x2a, 0x44, 0x98, 0x0a, 0x60, 0xd6, 0x42, 0x7a, 0xe9, 0xcd, 0x89, 0xa6, + 0x83, 0x56, 0x2d, 0x36, 0x59, 0x3d, 0x45, 0xd4, 0x47, 0x08, 0xc2, 0x44, 0x6b, 0xbb, 0xa2, 0x23, + 0xec, 0x1b, 0xfd, 0x99, 0x04, 0x0b, 0xa3, 0x3d, 0x2a, 0x1f, 0xae, 0xad, 0x37, 0xd7, 0x35, 0xaa, + 0x47, 0x6b, 0xf8, 0xb0, 0x5e, 0xcd, 0x4e, 0x2f, 0x4c, 0x5f, 0x4f, 0x2c, 0x6d, 0x4e, 0xde, 0x70, + 0xed, 0x0c, 0xc4, 0x9a, 0xe5, 0x3a, 0x87, 0xca, 0x99, 0x0d, 0xe7, 0x9a, 0x70, 0x6d, 0x22, 0xa8, + 0xa0, 0x0e, 0xc5, 0xb9, 0x0e, 0xcd, 0xc3, 0xcc, 0x03, 0xad, 0xdd, 0xc3, 0x62, 0xb4, 0x3c, 0x71, + 0x2b, 0xf4, 0x8e, 0x54, 0xb8, 0x02, 0x11, 0x3e, 0x31, 0x28, 0x05, 0xf1, 0x52, 0xad, 0xb9, 0xf4, + 0xed, 0xb7, 0x57, 0x2a, 0xeb, 0xf2, 0x94, 0x58, 0x82, 0x5f, 0x49, 0x70, 0xb9, 0xe9, 0x3a, 0x58, + 0xeb, 0xd4, 0xad, 0x3d, 0x4c, 0xe8, 0x98, 0xaa, 0xd8, 0xd5, 0xcc, 0x36, 0x41, 0xd7, 0x20, 0x4d, + 0x58, 0x8e, 0xaa, 0x19, 0x86, 0x83, 0x09, 0x11, 0x0d, 0xa6, 0xb8, 0xb4, 0xc4, 0x85, 0xe8, 0x9b, + 0x10, 0x26, 0x5d, 0xcd, 0x62, 0x2d, 0x27, 0x96, 0xae, 0x04, 0xa6, 0xcd, 0xdb, 0x39, 0xcd, 0xae, + 0x66, 0x95, 0xc3, 0x9f, 0xf7, 0xf3, 0x53, 0x0a, 0x2b, 0x8a, 0xca, 0x00, 0xc4, 0xd5, 0x1c, 0x57, + 0xa5, 0x14, 0x20, 0x94, 0xf6, 0xa5, 0x40, 0x45, 0x4a, 0x11, 0xc5, 0xfd, 0xb6, 0x5e, 0xdc, 0xf6, + 0x28, 0x42, 0x54, 0x8f, 0xb3, 0x6a, 0x54, 0x5a, 0xd0, 0xe0, 0xca, 0x48, 0xbf, 0xb7, 0x1c, 0x7b, + 0x8f, 0xf5, 0x68, 0x19, 0x92, 0x7a, 0xcf, 0xb5, 0x1f, 0x60, 0x87, 0x37, 0x20, 0x4d, 0xde, 0x40, + 0x42, 0x54, 0x64, 0x4d, 0xfc, 0x7d, 0x04, 0x52, 0x7c, 0xbf, 0x78, 0x53, 0x32, 0xdc, 0x71, 0xe9, + 0x59, 0x3a, 0x8e, 0x6e, 0x43, 0x0c, 0x5b, 0x06, 0x47, 0x08, 0x4d, 0x8e, 0x10, 0xc5, 0x96, 0xc1, + 0xea, 0xbf, 0xc0, 0x37, 0x20, 0x9d, 0xb5, 0x78, 0x39, 0x7a, 0xd4, 0xcf, 0x4f, 0xef, 0x28, 0x75, + 0xbe, 0x13, 0x7f, 0x24, 0xc1, 0x5c, 0xcf, 0x31, 0x89, 0xda, 0x3a, 0x54, 0xdb, 0xb6, 0xae, 0xb5, + 0x4d, 0xf7, 0x50, 0x3d, 0x78, 0x90, 0x9d, 0x61, 0x1a, 0x7d, 0xfb, 0x4c, 0x5a, 0x10, 0xc3, 0x2c, + 0xee, 0x38, 0x26, 0x29, 0x1f, 0xde, 0x11, 0x08, 0x6b, 0x0f, 0x98, 0xd6, 0x95, 0xe7, 0x8f, 0xfa, + 0x79, 0x79, 0x47, 0xa9, 0x07, 0xb3, 0xee, 0x29, 0x72, 0x6f, 0xa4, 0x30, 0x7a, 0x0f, 0x72, 0x06, + 0xee, 0x3a, 0x58, 0xd7, 0x28, 0x21, 0xb4, 0x18, 0xb2, 0xda, 0xd1, 0x2c, 0x73, 0x17, 0x13, 0x37, + 0x1b, 0x66, 0x0a, 0x9a, 0x1d, 0x94, 0xe0, 0x4d, 0xaf, 0x8b, 0x7c, 0xa4, 0xf9, 0x6c, 0x62, 0xda, + 0x96, 0x6a, 0x73, 0x7a, 0xca, 0x46, 0xd8, 0x44, 0x2d, 0x9d, 0x9f, 0xd8, 0x94, 0x0c, 0x1e, 0xa3, + 0x60, 0x05, 0x66, 0x03, 0x4d, 0x30, 0xe2, 0x8c, 0x33, 0xfc, 0x1b, 0x13, 0xef, 0x79, 0x25, 0x8d, + 0x87, 0x79, 0xeb, 0x8f, 0x25, 0xc8, 0x51, 0xeb, 0x83, 0x75, 0x3a, 0x68, 0xdf, 0xb4, 0xa9, 0x0e, + 0xd6, 0x6d, 0xc7, 0xc8, 0x46, 0xe9, 0xa8, 0xcb, 0xcd, 0x7f, 0x9d, 0xd4, 0xe8, 0x30, 0x23, 0xd9, + 0xeb, 0x99, 0x46, 0x71, 0x67, 0xa7, 0x5e, 0x3d, 0xea, 0xe7, 0xb3, 0x5b, 0x1e, 0xb8, 0xaf, 0x18, + 0x0a, 0x83, 0x56, 0xb2, 0xdd, 0x13, 0x72, 0xd0, 0x3b, 0x90, 0xd6, 0xed, 0x76, 0x1b, 0xeb, 0x6c, + 0x9c, 0x3b, 0x4a, 0x3d, 0x1b, 0x63, 0x4a, 0x93, 0x39, 0xea, 0xe7, 0x53, 0x15, 0x3f, 0x87, 0xaa, + 0x4f, 0x4a, 0x0f, 0x26, 0x73, 0x15, 0xb8, 0x74, 0xac, 0x0e, 0x9c, 0xc5, 0x3c, 0xf1, 0x20, 0xf3, + 0xc8, 0x90, 0xe6, 0x8b, 0xe2, 0x6d, 0xcc, 0xc2, 0x3f, 0x20, 0x48, 0x2b, 0x98, 0xb8, 0xb6, 0x83, + 0xbd, 0x1d, 0x15, 0xdc, 0x0d, 0xe1, 0x67, 0xd8, 0x0d, 0xbf, 0x94, 0x60, 0x8e, 0xba, 0x19, 0x8e, + 0xd9, 0x75, 0x6d, 0x47, 0x75, 0xf0, 0x43, 0xc7, 0x74, 0x31, 0xc9, 0x86, 0x98, 0xca, 0x97, 0x4e, + 0x59, 0xd0, 0xe1, 0x8e, 0x14, 0xab, 0x3e, 0x88, 0x22, 0x30, 0xb8, 0xd6, 0xdf, 0xfe, 0xe1, 0xbf, + 0xe5, 0x6f, 0x4d, 0xb4, 0x66, 0xe3, 0x9e, 0x4f, 0xb1, 0x5e, 0x55, 0x90, 0x31, 0x06, 0x8c, 0x5e, + 0x84, 0x30, 0xdd, 0x35, 0xcc, 0xd2, 0xc4, 0xcb, 0xb1, 0xa3, 0x7e, 0x3e, 0x4c, 0xf7, 0x95, 0xc2, + 0xa4, 0xc8, 0x85, 0x79, 0xb1, 0x69, 0xfc, 0x3d, 0xcc, 0x74, 0x34, 0xca, 0x86, 0xf4, 0xde, 0xe4, + 0x43, 0xe2, 0xb3, 0xef, 0x2d, 0x21, 0x33, 0xef, 0x7c, 0xf6, 0x50, 0x6b, 0x2c, 0x07, 0x6d, 0x41, + 0x9a, 0xda, 0xee, 0x96, 0x46, 0xb0, 0x4a, 0xbb, 0x4c, 0xb2, 0x32, 0x6b, 0x6f, 0x74, 0x4f, 0x90, + 0xfb, 0x6d, 0x5a, 0xa6, 0x58, 0x15, 0x85, 0x03, 0xf3, 0x96, 0x32, 0x02, 0x32, 0x82, 0x56, 0x20, + 0xe1, 0x6a, 0xad, 0xb6, 0x07, 0xc7, 0x49, 0xe8, 0x95, 0x13, 0xe0, 0xb6, 0x69, 0xc9, 0x00, 0x16, + 0xb8, 0x9e, 0x80, 0xa0, 0x2a, 0x80, 0x7b, 0xd8, 0xf5, 0x70, 0xd2, 0x0c, 0xe7, 0xda, 0x49, 0x38, + 0x87, 0xdd, 0x20, 0x4c, 0xdc, 0x15, 0x69, 0x82, 0x1a, 0x90, 0xe4, 0xce, 0xa5, 0xc0, 0x99, 0x65, + 0x38, 0xaf, 0x9e, 0x80, 0xc3, 0x6c, 0xa6, 0x16, 0x40, 0x4a, 0x10, 0x5f, 0x42, 0x50, 0x09, 0xa2, + 0xdc, 0xa9, 0x25, 0xd9, 0x14, 0x83, 0xf9, 0xfa, 0x49, 0xdd, 0x61, 0xa5, 0x02, 0x53, 0xef, 0xd5, + 0x43, 0x8b, 0x90, 0xa0, 0x86, 0xc6, 0x31, 0x0d, 0xac, 0x1a, 0x2d, 0x46, 0x70, 0xf1, 0x72, 0xfa, + 0xa8, 0x9f, 0x87, 0x4d, 0x21, 0xae, 0x96, 0x15, 0xf0, 0x8a, 0x54, 0x5b, 0xe8, 0x35, 0xc8, 0x74, + 0x1d, 0xdc, 0xd5, 0x1c, 0xac, 0xea, 0x76, 0xa7, 0xdb, 0xc6, 0x2e, 0x36, 0xd8, 0x86, 0x8e, 0x29, + 0xb2, 0xc8, 0xa8, 0x78, 0x72, 0x6e, 0xbb, 0x35, 0x97, 0xba, 0x85, 0x04, 0x3b, 0xb4, 0x64, 0x9c, + 0x95, 0x4c, 0x31, 0x69, 0x5d, 0x08, 0xd1, 0x21, 0x5c, 0x26, 0x87, 0xc4, 0xc5, 0x1d, 0x95, 0x4d, + 0x37, 0x51, 0x3b, 0xe6, 0x9e, 0x43, 0x49, 0x39, 0x9b, 0x61, 0xc3, 0xaa, 0x4c, 0xae, 0x6c, 0x4d, + 0x86, 0xc3, 0x96, 0x91, 0xac, 0x0b, 0x14, 0xee, 0xf8, 0xcc, 0x93, 0x63, 0xb2, 0xd0, 0x5b, 0x70, + 0x69, 0xb0, 0x33, 0x88, 0xda, 0xed, 0xb5, 0xda, 0x26, 0xd9, 0xc7, 0x46, 0x16, 0x58, 0x47, 0xe7, + 0x03, 0x99, 0x5b, 0x5e, 0x1e, 0x3a, 0x1c, 0xda, 0xec, 0x3a, 0x9d, 0x1d, 0x6d, 0x0f, 0x67, 0x13, + 0x0b, 0xd2, 0xf5, 0x99, 0xf2, 0xea, 0xd3, 0x7e, 0xbe, 0x3a, 0xf1, 0x4e, 0x25, 0xb8, 0xb3, 0xe8, + 0x3a, 0x18, 0x07, 0x36, 0x7e, 0x45, 0xe0, 0x05, 0xf7, 0xac, 0x27, 0x43, 0x0a, 0xc0, 0x80, 0xf2, + 0xb3, 0xc9, 0x67, 0xb6, 0x47, 0x01, 0x14, 0x64, 0x01, 0x72, 0xf0, 0x03, 0xad, 0x6d, 0x1a, 0x9a, + 0x8b, 0x55, 0xd3, 0x32, 0xf0, 0x23, 0x4c, 0xb2, 0x88, 0x4d, 0xfd, 0xbb, 0x93, 0x4f, 0xbd, 0xe2, + 0x63, 0xd4, 0x29, 0x84, 0xd0, 0xb4, 0x8c, 0x33, 0x2c, 0xc6, 0x24, 0xf7, 0x3f, 0x12, 0x64, 0xc6, + 0x78, 0x0e, 0x6d, 0x43, 0xc8, 0x3f, 0x20, 0x51, 0x53, 0x13, 0x62, 0x87, 0xa3, 0x8b, 0x70, 0x5e, + 0xc8, 0x34, 0xd0, 0x1e, 0xc4, 0xa9, 0x4a, 0x5a, 0x2e, 0x3d, 0x7d, 0x85, 0x18, 0x78, 0xe3, 0xa8, + 0x9f, 0x8f, 0x6d, 0x31, 0xe1, 0x85, 0x9b, 0x88, 0x71, 0xf0, 0xba, 0x81, 0xf2, 0x90, 0x70, 0x6d, + 0x15, 0x3f, 0x32, 0x89, 0x6b, 0x5a, 0x7b, 0xcc, 0x2f, 0x8a, 0x29, 0xe0, 0xda, 0x35, 0x21, 0xc9, + 0xfd, 0x79, 0x08, 0xd0, 0x38, 0x15, 0xa2, 0xbf, 0x93, 0xe0, 0x45, 0xcf, 0x59, 0xb2, 0x1d, 0x73, + 0xcf, 0xb4, 0xb4, 0xf6, 0x90, 0xd7, 0x24, 0xb1, 0x75, 0xf8, 0xf4, 0x22, 0x7c, 0x2b, 0x3c, 0xa9, + 0x4d, 0x01, 0x3f, 0xea, 0x51, 0xbd, 0x48, 0x0d, 0x3b, 0xf7, 0xa8, 0xc6, 0x8a, 0xdc, 0x53, 0xb2, + 0xbd, 0x13, 0x2a, 0xe7, 0xd6, 0xe0, 0xa5, 0x53, 0x81, 0xcf, 0x63, 0xa6, 0x73, 0x3f, 0x94, 0xe0, + 0xca, 0x09, 0xc6, 0x2f, 0x88, 0x93, 0xe2, 0x38, 0x77, 0x83, 0x38, 0x89, 0xa5, 0xef, 0x5e, 0xc0, + 0xc0, 0x06, 0x3b, 0xb1, 0x02, 0x2f, 0x9c, 0x48, 0x20, 0x67, 0x8d, 0x26, 0x16, 0x04, 0xfa, 0x17, + 0x09, 0x66, 0x47, 0xf6, 0x03, 0xfa, 0x24, 0xa0, 0xe0, 0x75, 0x7a, 0xbe, 0x65, 0x8d, 0x3c, 0x17, + 0x2d, 0x3f, 0x18, 0xd7, 0xf2, 0x0d, 0xda, 0x02, 0x6b, 0x98, 0xb5, 0xf0, 0xc1, 0x33, 0xb7, 0xc0, + 0x21, 0x06, 0x9a, 0xde, 0x08, 0xc7, 0x24, 0x39, 0x54, 0x78, 0x93, 0x0e, 0x90, 0xcd, 0xac, 0x7f, + 0xe0, 0x79, 0x09, 0x60, 0xdf, 0xdc, 0xdb, 0x57, 0x1f, 0x6a, 0x2e, 0x76, 0x44, 0x68, 0x21, 0x4e, + 0x25, 0x1f, 0x51, 0x41, 0xe1, 0x4f, 0x01, 0x52, 0xf5, 0x4e, 0xd7, 0x76, 0x5c, 0xcf, 0xeb, 0xba, + 0x03, 0x11, 0x4e, 0xf8, 0x42, 0xc9, 0x8b, 0xa7, 0x2c, 0xe3, 0x50, 0x4d, 0x6e, 0xa7, 0x05, 0xc3, + 0x08, 0x0c, 0xb4, 0x09, 0x51, 0x6e, 0x1c, 0x49, 0xf6, 0x0a, 0x83, 0x5b, 0x9c, 0x18, 0x8e, 0x9b, + 0x59, 0xcf, 0x36, 0x0a, 0x14, 0xdf, 0x3f, 0x0a, 0x1d, 0xeb, 0x1f, 0xbd, 0x0f, 0x11, 0x1e, 0x98, + 0x12, 0x27, 0xc7, 0xfc, 0x31, 0x47, 0xce, 0xfa, 0xe6, 0xb2, 0xd9, 0xc6, 0xcb, 0xac, 0x98, 0xd7, + 0x5b, 0x5e, 0x09, 0xbd, 0x02, 0x31, 0x42, 0x5c, 0x95, 0x98, 0x3f, 0xe0, 0x1e, 0xe7, 0x34, 0x8f, + 0x79, 0x34, 0x9b, 0xdb, 0x4d, 0xf3, 0x07, 0x58, 0x89, 0x12, 0xe2, 0xd2, 0x0f, 0x74, 0x15, 0x98, + 0xf5, 0x25, 0x1a, 0xb5, 0xa9, 0xcc, 0x7c, 0x4e, 0x2b, 0x01, 0x09, 0xc3, 0x39, 0x30, 0xbb, 0xea, + 0xee, 0x01, 0xe1, 0x36, 0x4b, 0xe0, 0x1c, 0x98, 0xdd, 0xe5, 0x35, 0xa2, 0x44, 0x69, 0xe6, 0xf2, + 0x01, 0x41, 0x39, 0x88, 0x3d, 0xd4, 0xda, 0x6d, 0xe6, 0xe1, 0xce, 0x30, 0x14, 0x3f, 0x3d, 0x4c, + 0x92, 0x91, 0xaf, 0x96, 0x24, 0x85, 0x4f, 0xd9, 0xd5, 0xdc, 0x7d, 0x76, 0x1c, 0x89, 0x2b, 0xc0, + 0x45, 0x5b, 0x9a, 0xbb, 0x8f, 0xb2, 0x10, 0xe5, 0xe3, 0x22, 0xd9, 0xd8, 0xc2, 0xf4, 0xf5, 0xa4, + 0xe2, 0x25, 0xd1, 0xab, 0x30, 0x6b, 0xb2, 0x23, 0xb6, 0x6a, 0x98, 0x0e, 0xd6, 0xdd, 0xf6, 0x21, + 0xb3, 0xb7, 0x31, 0x25, 0xcd, 0xc5, 0x55, 0x21, 0x45, 0x37, 0x40, 0x1e, 0x75, 0x50, 0x98, 0x9d, + 0x8c, 0x29, 0xb3, 0x23, 0xfe, 0x09, 0xf5, 0x65, 0xc4, 0x5a, 0x07, 0x0c, 0x7f, 0x96, 0xfb, 0x32, + 0x22, 0x63, 0x60, 0xf4, 0x6f, 0x80, 0x2c, 0xbc, 0x93, 0x41, 0xd9, 0x14, 0xc7, 0xe5, 0xf2, 0x41, + 0xd1, 0x22, 0xcc, 0x75, 0x35, 0x87, 0x60, 0xb5, 0xd5, 0xb3, 0x8c, 0x36, 0x56, 0x39, 0x56, 0x36, + 0xcd, 0x4a, 0x67, 0x58, 0x56, 0x99, 0xe5, 0x70, 0xbd, 0x3b, 0xeb, 0xd4, 0x76, 0xf9, 0xff, 0xe1, + 0xd4, 0x96, 0xfb, 0x45, 0x08, 0x66, 0xd8, 0x2e, 0x43, 0xb7, 0x20, 0x4c, 0xd7, 0x52, 0xc4, 0x19, + 0x26, 0xf5, 0x9c, 0x59, 0x1d, 0x84, 0x20, 0x6c, 0x69, 0x1d, 0x9c, 0x45, 0x6c, 0xa5, 0xd9, 0x37, + 0xba, 0x02, 0x51, 0x82, 0xef, 0xab, 0x0f, 0xb4, 0x76, 0x76, 0x8e, 0x29, 0x62, 0x84, 0xe0, 0xfb, + 0xf7, 0xb4, 0x36, 0xba, 0x04, 0x11, 0x93, 0xa8, 0x16, 0x7e, 0x98, 0x9d, 0xe7, 0x7c, 0x6a, 0x92, + 0x0d, 0xfc, 0x10, 0x7d, 0x0d, 0xe2, 0x0f, 0x35, 0xa2, 0xe2, 0x4e, 0xd7, 0x3d, 0x64, 0x73, 0x11, + 0xa3, 0xaa, 0x4b, 0x6a, 0x34, 0xcd, 0xcc, 0xae, 0xe6, 0xec, 0x61, 0x57, 0xd5, 0xed, 0x36, 0xc9, + 0x5e, 0xa2, 0x5b, 0x95, 0x7a, 0xed, 0x54, 0x54, 0xb1, 0xdb, 0xa4, 0x11, 0x8e, 0x85, 0xe4, 0xe9, + 0x46, 0x38, 0x36, 0x2d, 0x87, 0x1b, 0xe1, 0x58, 0x58, 0x9e, 0x69, 0x84, 0x63, 0x33, 0x72, 0xa4, + 0x11, 0x8e, 0x45, 0xe4, 0x68, 0x23, 0x1c, 0x8b, 0xca, 0xb1, 0x46, 0x38, 0x16, 0x93, 0xe3, 0x8d, + 0x70, 0x2c, 0x2e, 0x43, 0x23, 0x1c, 0x03, 0x39, 0xd1, 0x08, 0xc7, 0x12, 0x72, 0xb2, 0x11, 0x8e, + 0x25, 0xe5, 0x54, 0x23, 0x1c, 0x4b, 0xc9, 0xe9, 0x46, 0x38, 0x96, 0x96, 0x67, 0x1b, 0xe1, 0xd8, + 0xac, 0x2c, 0x37, 0xc2, 0x31, 0x59, 0xce, 0x34, 0xc2, 0xb1, 0x8c, 0x8c, 0x72, 0x35, 0x11, 0xd8, + 0xd2, 0xd0, 0x77, 0x87, 0xe6, 0x69, 0x62, 0x8f, 0x9e, 0x55, 0x2a, 0xfc, 0x52, 0x02, 0xb9, 0x89, + 0xef, 0xf7, 0xb0, 0xa5, 0xe3, 0x7b, 0x5a, 0xbb, 0xb2, 0xdf, 0xb3, 0x0e, 0xd0, 0x2b, 0x30, 0xab, + 0xd3, 0x0f, 0x95, 0x47, 0x7b, 0xe8, 0x8c, 0x49, 0x6c, 0xc6, 0x52, 0x4c, 0xdc, 0xa4, 0x52, 0x3a, + 0x71, 0x2f, 0x01, 0x88, 0x72, 0x94, 0x4d, 0x78, 0x24, 0x38, 0xce, 0x8b, 0x50, 0x0a, 0x19, 0x81, + 0x71, 0xec, 0x87, 0x8c, 0xb2, 0x86, 0x60, 0x14, 0xfb, 0x21, 0x5a, 0x84, 0x79, 0x0b, 0x3f, 0x72, + 0xd5, 0xd1, 0xc2, 0x8c, 0x9e, 0x94, 0x0c, 0xcd, 0xab, 0x04, 0x2b, 0x14, 0xfe, 0x39, 0x04, 0xb3, + 0x5e, 0xa7, 0x3d, 0x4e, 0xdf, 0x05, 0x99, 0xae, 0xae, 0x69, 0xa8, 0xae, 0xcd, 0x91, 0x3c, 0x76, + 0x7f, 0xff, 0x14, 0x3a, 0x1e, 0x41, 0xa1, 0xe9, 0xba, 0xb1, 0x6d, 0xb3, 0xe6, 0xb8, 0x13, 0xa0, + 0xa4, 0x48, 0x50, 0x96, 0xdb, 0x81, 0xb4, 0x57, 0x89, 0x4b, 0x50, 0x05, 0x22, 0x43, 0xed, 0xbd, + 0x36, 0x41, 0x7b, 0xde, 0x54, 0x2b, 0xa2, 0x6a, 0xee, 0x77, 0x01, 0x8d, 0xb7, 0x1d, 0x34, 0xfd, + 0x33, 0xdc, 0xf4, 0x6f, 0x0e, 0x3b, 0x20, 0xef, 0x9e, 0x6f, 0x6c, 0x81, 0x6e, 0x07, 0x43, 0x15, + 0xff, 0x18, 0x82, 0x34, 0x37, 0x4c, 0xbe, 0x4d, 0xa5, 0x14, 0x45, 0x19, 0xd0, 0xb4, 0xf6, 0xd4, + 0xae, 0x10, 0xb2, 0xf1, 0x85, 0x14, 0xd9, 0xcb, 0xf0, 0x0b, 0x7f, 0x03, 0x52, 0x0e, 0xd6, 0x8c, + 0x41, 0xc1, 0x10, 0x2b, 0x98, 0xa4, 0x42, 0xbf, 0xd0, 0x35, 0x48, 0x33, 0xbf, 0x67, 0x50, 0x6a, + 0x9a, 0x95, 0x4a, 0x31, 0xa9, 0x5f, 0xac, 0x0c, 0x29, 0xd2, 0xd5, 0xac, 0x41, 0xa9, 0x30, 0x9b, + 0xd4, 0x33, 0x02, 0xab, 0x49, 0x5a, 0x27, 0xe8, 0x10, 0x38, 0x98, 0xf4, 0x3a, 0x58, 0xed, 0xda, + 0xfc, 0xe4, 0x3d, 0xad, 0xc4, 0xb9, 0x64, 0xcb, 0x26, 0x68, 0x87, 0xa9, 0x0a, 0x9b, 0x0b, 0xd5, + 0xe0, 0x93, 0x93, 0x8d, 0xb0, 0x56, 0x6e, 0x4e, 0x3e, 0x9d, 0xca, 0x2c, 0x19, 0x16, 0x14, 0xfe, + 0x5a, 0x82, 0x2b, 0xf4, 0xfc, 0xcd, 0x77, 0x5a, 0x85, 0xdd, 0xe1, 0x78, 0xda, 0xa9, 0x41, 0x94, + 0x9d, 0xe1, 0x7d, 0x47, 0x6c, 0xf5, 0xa8, 0x9f, 0x8f, 0xd0, 0xd2, 0x17, 0xb6, 0x72, 0x11, 0x0a, + 0x5c, 0x67, 0x27, 0x4a, 0xd7, 0xd1, 0x2c, 0x62, 0xd2, 0xb3, 0x15, 0x5d, 0xb6, 0x0e, 0xee, 0xb4, + 0xb0, 0xc3, 0x17, 0x23, 0xa9, 0xcc, 0x0f, 0x65, 0xae, 0xf3, 0xbc, 0x42, 0x0e, 0xb2, 0xa3, 0x5d, + 0xf6, 0xc3, 0x55, 0xbf, 0x05, 0x97, 0x37, 0xf0, 0xc3, 0xe3, 0x46, 0x53, 0x86, 0x28, 0x67, 0x3a, + 0x4f, 0xe5, 0xaf, 0x8f, 0x92, 0x4e, 0xf0, 0x1a, 0xab, 0xc8, 0x7a, 0xba, 0xcd, 0x2a, 0x28, 0x5e, + 0xc5, 0xc2, 0xa7, 0x70, 0x65, 0x04, 0xdd, 0x5f, 0xbe, 0x0f, 0x20, 0x42, 0xcf, 0xe9, 0xc2, 0x3d, + 0x4b, 0x8f, 0x53, 0xda, 0x38, 0x7a, 0x93, 0x96, 0x57, 0x44, 0xb5, 0x82, 0xc2, 0xe2, 0x6c, 0xbd, + 0x0e, 0xa6, 0x1a, 0x72, 0xc7, 0x24, 0x2e, 0xfa, 0x10, 0x92, 0x42, 0x23, 0xa8, 0xa2, 0x78, 0xdd, + 0x3e, 0x43, 0xa9, 0x12, 0x8e, 0x0f, 0x42, 0x0a, 0x7f, 0x23, 0xc1, 0x5c, 0xd5, 0xb1, 0xbb, 0x5d, + 0x6c, 0x08, 0x93, 0xc3, 0xe7, 0xc2, 0xb3, 0x34, 0x52, 0xc0, 0xd2, 0x6c, 0x40, 0xa8, 0x5e, 0x15, + 0xfe, 0xf0, 0xed, 0x8b, 0xba, 0xd9, 0xf5, 0x2a, 0x7a, 0x97, 0x4f, 0x48, 0x8f, 0x30, 0xfe, 0x4c, + 0x8f, 0x85, 0x5b, 0x86, 0xd4, 0x94, 0x15, 0x54, 0x44, 0x85, 0xc2, 0xcf, 0xa3, 0x70, 0x29, 0x38, + 0xc9, 0x2b, 0x15, 0xaf, 0xe3, 0x9f, 0x41, 0xd4, 0x3b, 0x72, 0x4f, 0xc0, 0x93, 0xc7, 0x41, 0x14, + 0xc5, 0x7c, 0x04, 0x8f, 0xdd, 0x1e, 0x26, 0x6a, 0x42, 0xc6, 0xb4, 0x5c, 0xec, 0xb4, 0xb1, 0xf6, + 0x80, 0x3a, 0x17, 0x74, 0xce, 0x44, 0x88, 0x73, 0x52, 0x53, 0x2e, 0x07, 0x00, 0xb8, 0x4b, 0xf0, + 0x19, 0xcc, 0x05, 0x41, 0xbd, 0xfe, 0x9f, 0x1e, 0x5b, 0x63, 0xdd, 0x1b, 0xc0, 0x7a, 0x41, 0xc0, + 0x00, 0x90, 0x08, 0x10, 0xa0, 0x8f, 0xfd, 0x73, 0x01, 0x8f, 0x9f, 0xde, 0x7a, 0xe6, 0x19, 0xa9, + 0x8e, 0x9c, 0x11, 0x86, 0x3c, 0x5d, 0x66, 0x04, 0xbf, 0x22, 0x4f, 0xf7, 0x1e, 0x44, 0x78, 0x88, + 0x4d, 0xdc, 0x19, 0xdc, 0x7e, 0xd6, 0x21, 0xf0, 0xd0, 0x9d, 0x22, 0xd0, 0x72, 0x7f, 0x22, 0x41, + 0x32, 0xb8, 0xdc, 0xc8, 0x84, 0x18, 0x9b, 0x7e, 0x8f, 0xd2, 0xa6, 0x9f, 0xfb, 0xc9, 0x8f, 0xab, + 0x52, 0xdd, 0xa0, 0x8e, 0x98, 0xe1, 0xd8, 0xdd, 0xc1, 0x9d, 0xd1, 0xb4, 0x12, 0xa3, 0x02, 0xea, + 0x3a, 0xe6, 0x7e, 0x0f, 0xe2, 0xfe, 0xa4, 0x07, 0x62, 0x39, 0xd3, 0xcf, 0x31, 0x96, 0x73, 0x6a, + 0xfb, 0x55, 0x48, 0x0d, 0xcd, 0x18, 0xba, 0xec, 0xf7, 0x21, 0x5c, 0x8e, 0xf0, 0x3e, 0x9c, 0x89, + 0x52, 0xf8, 0xab, 0x28, 0xcc, 0x1d, 0xc7, 0xb4, 0x9f, 0x80, 0x1c, 0xe0, 0x2d, 0xb5, 0x6d, 0x12, + 0x57, 0xe8, 0xe6, 0x8d, 0xd3, 0x43, 0x0f, 0x01, 0xf2, 0x13, 0xaa, 0x98, 0x76, 0x86, 0x29, 0xf1, + 0x7b, 0x90, 0x36, 0x78, 0xc7, 0x45, 0xf4, 0x53, 0xdc, 0xfc, 0x9e, 0x76, 0x18, 0x3e, 0x86, 0x00, + 0x05, 0x7a, 0xca, 0x08, 0x64, 0x11, 0xa4, 0x43, 0xca, 0x07, 0x3f, 0xec, 0x8a, 0x53, 0xd5, 0xc5, + 0xc9, 0x30, 0xe9, 0xb5, 0x42, 0x31, 0xd1, 0x1e, 0xcc, 0x7a, 0x8d, 0x78, 0x07, 0xf0, 0xf8, 0x73, + 0x69, 0xc6, 0x9b, 0x98, 0xa6, 0x38, 0x90, 0xff, 0x48, 0x82, 0x39, 0xaf, 0x25, 0xff, 0x96, 0xc0, + 0x34, 0x18, 0x9d, 0xa5, 0xca, 0xcd, 0xa3, 0x7e, 0x3e, 0x23, 0x66, 0xc6, 0xbb, 0x16, 0xb8, 0xb0, + 0xde, 0x65, 0x8c, 0x11, 0x40, 0x83, 0xfa, 0x10, 0x34, 0x9f, 0x36, 0x3c, 0x33, 0xf0, 0x21, 0x28, + 0xb1, 0x5d, 0xdc, 0x87, 0xa0, 0x9f, 0x75, 0x03, 0xfd, 0xa1, 0x04, 0x19, 0x7e, 0x69, 0xd1, 0xe9, + 0xb9, 0x1a, 0xbf, 0x1f, 0xf4, 0x4e, 0xe6, 0x9f, 0x1c, 0xf5, 0xf3, 0xb3, 0x6c, 0x79, 0xd7, 0x45, + 0x1e, 0x6b, 0xb6, 0xfc, 0xac, 0xcd, 0x0e, 0x50, 0xc4, 0x41, 0xd6, 0x17, 0x18, 0x68, 0x0d, 0xd2, + 0x3c, 0x5c, 0xe1, 0x3d, 0x6d, 0x61, 0x47, 0xf6, 0x54, 0xf9, 0xe5, 0xa7, 0xfd, 0xfc, 0xc2, 0x31, + 0xfb, 0x84, 0x47, 0x3a, 0xee, 0xf1, 0xb2, 0x4a, 0x6a, 0x37, 0x98, 0x44, 0x77, 0x60, 0x96, 0x3b, + 0x9e, 0xfe, 0x01, 0x97, 0x05, 0x2c, 0x26, 0xbc, 0x6a, 0xe3, 0x4e, 0xab, 0x2f, 0x15, 0x51, 0xa8, + 0xcb, 0x30, 0x7f, 0xac, 0xcf, 0xf4, 0x45, 0x04, 0x2e, 0x0f, 0xd3, 0xaa, 0xef, 0xd5, 0xa8, 0xa3, + 0xf6, 0xf6, 0x83, 0x89, 0xa9, 0xd9, 0xc3, 0xe0, 0xd4, 0xe8, 0xa5, 0x46, 0x2d, 0xee, 0x67, 0x23, + 0xd6, 0xeb, 0x19, 0xf0, 0xd9, 0xf2, 0x8e, 0xe0, 0x7b, 0x26, 0xec, 0x63, 0xdf, 0xb2, 0xf0, 0xb8, + 0xd3, 0x87, 0xcf, 0x00, 0xcf, 0xea, 0x7b, 0x49, 0xdf, 0xb6, 0xfc, 0x93, 0x04, 0xa9, 0xa1, 0x91, + 0xfd, 0x26, 0x8d, 0xcb, 0x96, 0xef, 0x5b, 0xf1, 0x07, 0x48, 0xef, 0x9c, 0x7f, 0x58, 0xc3, 0x2e, + 0x57, 0xee, 0x6f, 0x25, 0x48, 0x0d, 0x4d, 0xe4, 0x57, 0x64, 0x96, 0x9e, 0x7f, 0xcf, 0x5b, 0x90, + 0x1e, 0x5e, 0xa2, 0x40, 0x1b, 0xd2, 0xf3, 0x69, 0xa3, 0xf0, 0x1d, 0x88, 0x70, 0x09, 0x42, 0x90, + 0xfe, 0xa8, 0x54, 0xdf, 0xae, 0x6f, 0xac, 0xa8, 0xcb, 0x9b, 0x8a, 0xba, 0x52, 0x91, 0xa7, 0x50, + 0x12, 0x62, 0xd5, 0xda, 0x9d, 0x1a, 0x15, 0xca, 0x12, 0x4a, 0x40, 0x94, 0xa5, 0x6a, 0x55, 0x39, + 0x54, 0x28, 0x83, 0xcc, 0xb1, 0x77, 0x31, 0x35, 0x33, 0xf4, 0x14, 0x81, 0x8a, 0x30, 0xc7, 0x5c, + 0xfe, 0x0e, 0xf5, 0xac, 0xe8, 0xf6, 0x56, 0x03, 0xbe, 0x78, 0xc6, 0xcf, 0xa2, 0xbb, 0x77, 0x43, + 0xeb, 0xe0, 0xc2, 0xaf, 0xc2, 0x90, 0x19, 0x80, 0x78, 0x46, 0xf6, 0x2f, 0xa5, 0xc1, 0x79, 0x26, + 0x72, 0xe6, 0xed, 0xd3, 0x58, 0x7d, 0x71, 0xb4, 0x11, 0x17, 0xe6, 0x1f, 0xd1, 0x4d, 0xf3, 0xb4, + 0x9f, 0xcf, 0x8c, 0x76, 0x96, 0x5c, 0xf0, 0x26, 0xdd, 0xeb, 0x22, 0x8b, 0xbc, 0x9a, 0xd6, 0x81, + 0x3a, 0x78, 0x06, 0xc3, 0x23, 0xaf, 0xa6, 0x75, 0xb0, 0xa3, 0xd4, 0x95, 0x28, 0xcd, 0xdc, 0x71, + 0x4c, 0xd4, 0x80, 0xb0, 0xdd, 0x75, 0xbd, 0x03, 0xf4, 0xdb, 0xe7, 0x1a, 0xd2, 0x66, 0x57, 0x8c, + 0x47, 0x61, 0x18, 0xa8, 0xc1, 0x2f, 0x54, 0x07, 0x13, 0xcd, 0x08, 0x79, 0x42, 0x0a, 0x4d, 0x0d, + 0x2d, 0x44, 0x6e, 0x0f, 0x92, 0xc1, 0x19, 0x3b, 0xe6, 0x96, 0xa5, 0x34, 0x1c, 0xe4, 0x78, 0x6d, + 0xa2, 0xae, 0x8b, 0x03, 0x66, 0xe0, 0x32, 0xe4, 0x3b, 0x10, 0xf7, 0xc7, 0x71, 0x9e, 0x3b, 0x21, + 0xce, 0xf1, 0x7e, 0x04, 0x6f, 0x46, 0x8e, 0x14, 0xfa, 0x21, 0x48, 0x2a, 0x98, 0xd8, 0xed, 0x07, + 0xd8, 0xa0, 0x1e, 0x94, 0xff, 0xec, 0x4b, 0x9a, 0xfc, 0xd9, 0x57, 0x09, 0xe2, 0x03, 0x0b, 0x74, + 0x8e, 0xa7, 0x4f, 0x83, 0x5a, 0xe8, 0x36, 0x7c, 0x2d, 0xf8, 0xb6, 0xc8, 0xee, 0x59, 0x86, 0xe6, + 0x1c, 0xaa, 0x0e, 0xd6, 0xf4, 0x7d, 0x6c, 0x88, 0xcb, 0xbf, 0x17, 0x02, 0x8f, 0x8b, 0x44, 0x09, + 0x85, 0x17, 0x40, 0x9f, 0x40, 0xca, 0xaf, 0x44, 0xfd, 0x32, 0xe6, 0xc1, 0xa4, 0x97, 0xbe, 0x75, + 0xba, 0x2f, 0xe9, 0x8f, 0xba, 0xe8, 0xe1, 0x51, 0xff, 0x4b, 0x49, 0xb6, 0x02, 0xa9, 0xc2, 0xfb, + 0x90, 0x0c, 0xe6, 0xa2, 0x18, 0x84, 0x37, 0x36, 0x37, 0x6a, 0x7c, 0x4f, 0x97, 0x4b, 0x95, 0xb5, + 0xe5, 0xfa, 0x9d, 0x3b, 0xb2, 0x44, 0xe5, 0xb5, 0x8f, 0xeb, 0xdb, 0x72, 0x88, 0xee, 0x6e, 0xa5, + 0xd6, 0xdc, 0x2e, 0x29, 0xdb, 0xf2, 0x74, 0x01, 0x43, 0x2a, 0xd8, 0x12, 0xe5, 0x4c, 0xea, 0xb0, + 0x32, 0xc1, 0xd0, 0x99, 0xfd, 0xd5, 0x09, 0xfb, 0xea, 0xe9, 0x9e, 0x13, 0x44, 0x2d, 0xfc, 0x38, + 0x04, 0x68, 0xa0, 0x32, 0x01, 0x82, 0x1e, 0x6d, 0x2c, 0x74, 0xf1, 0xc6, 0xd0, 0x4f, 0x4e, 0x0f, + 0xaf, 0x4f, 0xb3, 0xf0, 0x3a, 0xe3, 0x8b, 0xdf, 0x68, 0x88, 0x5d, 0xb8, 0x30, 0xff, 0x1d, 0x06, + 0x54, 0x71, 0xb0, 0xe6, 0x62, 0xca, 0xc7, 0xe4, 0xb4, 0x78, 0x46, 0x19, 0x66, 0xf8, 0xf9, 0x3d, + 0x74, 0x9e, 0xf3, 0xbb, 0x98, 0x14, 0x5e, 0x15, 0x7d, 0x1f, 0x92, 0xba, 0xdd, 0xee, 0x75, 0x2c, + 0x95, 0xbd, 0xc1, 0x10, 0x87, 0x8d, 0x6f, 0x9f, 0xb6, 0xb5, 0xc7, 0x3a, 0x57, 0xac, 0xd8, 0x6d, + 0x9a, 0xf6, 0x5f, 0x2b, 0x32, 0x40, 0x56, 0x02, 0xbd, 0x08, 0x71, 0x9f, 0x66, 0x98, 0x5a, 0xc7, + 0x95, 0x81, 0x00, 0x2d, 0xc1, 0x8c, 0x46, 0x54, 0x7b, 0x97, 0x79, 0xce, 0x67, 0xed, 0x3b, 0x25, + 0xac, 0x91, 0xcd, 0x5d, 0x74, 0x13, 0x32, 0x1d, 0xed, 0x91, 0xba, 0xeb, 0x68, 0xba, 0xf0, 0x84, + 0xdb, 0x9c, 0xf6, 0x24, 0x65, 0xb6, 0xa3, 0x3d, 0x5a, 0x16, 0xf2, 0xba, 0xd1, 0xc6, 0xe8, 0x2d, + 0x48, 0xed, 0xde, 0xe7, 0xe7, 0x28, 0x6e, 0x82, 0xf8, 0x83, 0x96, 0xd9, 0xa3, 0x7e, 0x3e, 0xb1, + 0x7c, 0x97, 0x4d, 0x0c, 0x35, 0x40, 0x4a, 0x62, 0xf7, 0xbe, 0x9f, 0xc8, 0xfd, 0x97, 0x04, 0x51, + 0x31, 0x22, 0xd4, 0x05, 0x10, 0xd3, 0x63, 0x1a, 0x5c, 0xd5, 0x53, 0xe5, 0xbb, 0x47, 0xfd, 0x7c, + 0xbc, 0xc2, 0xa4, 0xf5, 0x2a, 0x79, 0xda, 0xcf, 0x7f, 0xf8, 0xac, 0xe6, 0xc3, 0x03, 0x51, 0xe2, + 0xbc, 0x91, 0xba, 0xc1, 0x82, 0xb6, 0xfb, 0x1a, 0x51, 0xf7, 0x4d, 0xe2, 0xda, 0x7b, 0x8e, 0xd6, + 0x11, 0x97, 0xc9, 0xc9, 0x7d, 0x8d, 0xac, 0x7a, 0x32, 0x94, 0xa3, 0x8e, 0xd8, 0x03, 0xfe, 0x84, + 0x86, 0xb3, 0x8b, 0x9f, 0x46, 0x4b, 0x70, 0xc9, 0xaf, 0xac, 0xd2, 0x99, 0x6a, 0xf5, 0xf4, 0x03, + 0xcc, 0x0c, 0x0e, 0x65, 0xf2, 0x39, 0x3f, 0x73, 0x5d, 0x7b, 0x54, 0xe6, 0x59, 0x85, 0x4b, 0x30, + 0x17, 0x58, 0x56, 0xdf, 0x6d, 0xc6, 0x20, 0xf3, 0x3b, 0xef, 0xc0, 0xfb, 0xdb, 0xbb, 0x30, 0x3b, + 0xf2, 0xbe, 0x5d, 0x90, 0x6d, 0x30, 0xd8, 0x38, 0xfc, 0x20, 0xbe, 0x58, 0xe1, 0x49, 0xef, 0x20, + 0x90, 0xd6, 0x87, 0xd2, 0x85, 0x39, 0xc8, 0xf8, 0xcd, 0xf8, 0x6d, 0xff, 0x6f, 0x12, 0xa2, 0x5b, + 0xda, 0x61, 0xdb, 0xd6, 0x0c, 0xb4, 0x00, 0x09, 0xef, 0xed, 0x8b, 0xd7, 0x5e, 0x5c, 0x09, 0x8a, + 0x86, 0xd5, 0x4c, 0x66, 0xb7, 0x3e, 0x01, 0x35, 0x33, 0x21, 0xdd, 0x23, 0xd8, 0xa1, 0x1a, 0xa0, + 0xb2, 0xc7, 0xf8, 0xdc, 0xb6, 0x94, 0xcb, 0x4f, 0xfb, 0xf9, 0xdb, 0x93, 0x2d, 0x1e, 0xd6, 0x7b, + 0x8e, 0xe9, 0x1e, 0x16, 0x9b, 0x77, 0xef, 0xec, 0x08, 0x28, 0xba, 0xc5, 0x6d, 0x25, 0xd5, 0x0b, + 0x26, 0xc5, 0x13, 0x27, 0xba, 0x10, 0x6a, 0xc7, 0xd4, 0x1d, 0x9b, 0x78, 0x77, 0x2b, 0x42, 0xba, + 0xce, 0x84, 0xe8, 0x55, 0x98, 0xdd, 0x35, 0x2d, 0x76, 0x3d, 0xe8, 0x95, 0xe3, 0xd7, 0x2a, 0x69, + 0x4f, 0x2c, 0x0a, 0x3e, 0x80, 0x74, 0xe0, 0x6d, 0x11, 0x55, 0xc2, 0x08, 0x53, 0xc2, 0xcd, 0xa3, + 0x7e, 0x3e, 0x35, 0xd8, 0xd4, 0x5c, 0x11, 0x2f, 0xe2, 0xc7, 0xa4, 0x06, 0xcd, 0x50, 0x35, 0x9c, + 0x87, 0x19, 0xf6, 0x53, 0x0d, 0xfe, 0x38, 0x53, 0xe1, 0x09, 0x54, 0x83, 0x94, 0x88, 0x7b, 0xf0, + 0xdf, 0x71, 0x88, 0x07, 0x59, 0x0b, 0x81, 0xa5, 0xf7, 0x7e, 0xe9, 0x51, 0xac, 0x59, 0xba, 0x6d, + 0x60, 0xa3, 0x46, 0xd3, 0x8a, 0x08, 0xf3, 0xb2, 0x04, 0x41, 0x2b, 0x90, 0xd6, 0xdb, 0x58, 0xb3, + 0x7a, 0x5d, 0x0f, 0x07, 0x4d, 0x88, 0x93, 0x12, 0xf5, 0x04, 0xd0, 0x06, 0xa0, 0x5d, 0xf6, 0x42, + 0x26, 0xd8, 0x2b, 0x76, 0x8d, 0x38, 0x09, 0x98, 0xcc, 0xea, 0x2a, 0x83, 0x9e, 0xa1, 0xb7, 0x61, + 0xa6, 0x8d, 0x35, 0x82, 0xc5, 0xcb, 0xdb, 0x85, 0x53, 0x68, 0x90, 0xfd, 0x96, 0x43, 0xe1, 0xc5, + 0xd1, 0xcb, 0x90, 0xb2, 0x6c, 0x4b, 0xd7, 0x2c, 0x1d, 0xb7, 0x19, 0x23, 0xf3, 0x1b, 0xcb, 0x61, + 0x21, 0x2a, 0x43, 0x84, 0xdf, 0x6d, 0x8b, 0x83, 0xee, 0xf5, 0x49, 0x9f, 0x3e, 0xaf, 0x4e, 0x29, + 0xa2, 0x26, 0xaa, 0x41, 0xd4, 0xe1, 0xef, 0x2c, 0xd8, 0x7d, 0xf7, 0x99, 0x01, 0xa7, 0xc0, 0x5b, + 0x97, 0xd5, 0x29, 0xc5, 0xab, 0x8b, 0xb6, 0xbd, 0x67, 0x87, 0xdc, 0xea, 0x8a, 0x97, 0x63, 0xc5, + 0x09, 0x8f, 0x11, 0x03, 0xc0, 0x21, 0x14, 0x3a, 0x40, 0x93, 0xdd, 0x57, 0xb1, 0x9b, 0xf0, 0xd3, + 0x07, 0x38, 0xf4, 0xe2, 0x82, 0x0e, 0x90, 0xd7, 0x44, 0x1b, 0x00, 0xba, 0xef, 0x09, 0xb0, 0x3b, + 0xf2, 0xc4, 0xd2, 0xeb, 0xe7, 0x71, 0x92, 0x57, 0xa7, 0x94, 0x00, 0x02, 0xba, 0x0b, 0x09, 0x7d, + 0x40, 0x6d, 0xd9, 0x59, 0x06, 0xf8, 0xc6, 0xb9, 0xec, 0xdb, 0x2a, 0xb5, 0x69, 0x03, 0x29, 0xfa, + 0x14, 0xd2, 0x64, 0xe8, 0x50, 0x95, 0xbd, 0xc4, 0x50, 0xdf, 0x3c, 0x6f, 0x50, 0x77, 0x75, 0x4a, + 0x19, 0x41, 0x42, 0xbf, 0x0d, 0xb2, 0x3b, 0x72, 0xf3, 0xc3, 0x2e, 0xb9, 0x4f, 0x7f, 0xd6, 0x77, + 0xc2, 0xfd, 0xd6, 0xea, 0x94, 0x32, 0x86, 0x86, 0x3e, 0x83, 0x59, 0x32, 0xfc, 0x13, 0x85, 0xec, + 0x15, 0xd6, 0xc0, 0x37, 0x4f, 0xbd, 0xbe, 0x38, 0xee, 0xc7, 0x18, 0xab, 0x53, 0xca, 0x28, 0x16, + 0x85, 0xb7, 0x86, 0x2f, 0x90, 0xd8, 0x13, 0x8a, 0xd3, 0xe1, 0x8f, 0xbf, 0xd0, 0xa2, 0xf0, 0x23, + 0x58, 0x68, 0x0d, 0xe2, 0x1d, 0xcf, 0x56, 0x64, 0x5f, 0x38, 0xf3, 0x1c, 0x32, 0x6a, 0xbe, 0x56, + 0xa7, 0x94, 0x41, 0xfd, 0x72, 0x1c, 0xa2, 0xe2, 0xa2, 0xd1, 0x7f, 0x05, 0xc0, 0xef, 0xff, 0x73, + 0xf2, 0xd7, 0x0a, 0x3f, 0x8d, 0x43, 0xcc, 0xf7, 0x44, 0x17, 0x01, 0xf9, 0x0e, 0xc7, 0xe0, 0x9d, + 0x2b, 0x35, 0x44, 0xa1, 0xd5, 0x29, 0x25, 0xe3, 0xe5, 0x0d, 0x9e, 0xba, 0xde, 0x1e, 0x7a, 0xfc, + 0x34, 0xc9, 0x8f, 0x49, 0x68, 0xd7, 0xfc, 0xd7, 0x51, 0xd4, 0x40, 0x74, 0x6c, 0xc3, 0xdc, 0x35, + 0x07, 0x06, 0x82, 0x07, 0xa7, 0xd3, 0x9e, 0x58, 0x18, 0x88, 0x6b, 0x90, 0x76, 0x7a, 0x16, 0xbb, + 0x59, 0x14, 0x21, 0x01, 0xee, 0x65, 0xa5, 0x84, 0x54, 0x9c, 0xea, 0x2b, 0x23, 0xdc, 0x73, 0xe3, + 0x4c, 0xee, 0xf1, 0xc6, 0xbe, 0x2a, 0xf9, 0xe4, 0xb3, 0x3c, 0x4a, 0x3e, 0x37, 0xcf, 0x26, 0x9f, + 0x00, 0x8c, 0xcf, 0x3e, 0x3b, 0xc7, 0xb2, 0xcf, 0xe2, 0x84, 0xdb, 0x27, 0x80, 0x38, 0x4c, 0x3f, + 0x95, 0x11, 0xfa, 0xb9, 0x71, 0x26, 0xfd, 0x04, 0xc7, 0x28, 0xf8, 0x67, 0xf3, 0x18, 0xfe, 0x79, + 0x63, 0x22, 0xfe, 0x09, 0x80, 0x05, 0x09, 0x48, 0x39, 0x8e, 0x80, 0x8a, 0x93, 0x11, 0x50, 0x00, + 0x72, 0x88, 0x81, 0xbe, 0x37, 0xc6, 0x40, 0xf2, 0xd9, 0x5b, 0xf8, 0xd8, 0x38, 0xd0, 0xaa, 0x34, + 0x46, 0x41, 0xda, 0x31, 0x14, 0x94, 0x61, 0xf0, 0x6f, 0x9d, 0x83, 0x82, 0x02, 0x0d, 0x8c, 0x73, + 0xd0, 0xc7, 0x90, 0x0c, 0xf2, 0x06, 0x7b, 0x0f, 0x74, 0x3a, 0xc3, 0x9d, 0xf0, 0xab, 0x2a, 0xa6, + 0x03, 0x81, 0x2c, 0xf4, 0xfd, 0x71, 0xfa, 0x99, 0x3b, 0x13, 0xfc, 0x84, 0x1b, 0xef, 0x55, 0x69, + 0x9c, 0x7f, 0xee, 0x04, 0xf9, 0x67, 0xfe, 0x4c, 0xeb, 0x34, 0xe6, 0xd7, 0xae, 0x4a, 0x01, 0x02, + 0x42, 0xaf, 0x40, 0xcc, 0x75, 0x34, 0x9d, 0xdd, 0x5a, 0x5c, 0x62, 0x57, 0x53, 0x2c, 0x66, 0xb4, + 0x4d, 0x65, 0x2c, 0xb6, 0xc4, 0x3e, 0x8c, 0x32, 0x40, 0xcc, 0x7b, 0x78, 0x11, 0x20, 0xad, 0xc2, + 0xcf, 0x24, 0x98, 0x6e, 0xd8, 0x2d, 0xf4, 0x52, 0x20, 0x90, 0x99, 0x12, 0x07, 0xd7, 0x99, 0x86, + 0xdd, 0x12, 0x11, 0xc9, 0x0f, 0x06, 0xb5, 0xc5, 0x51, 0xf1, 0x1b, 0xa7, 0x74, 0xd9, 0x8f, 0x03, + 0xfb, 0x95, 0xd0, 0x7b, 0x10, 0xed, 0x72, 0x57, 0x5c, 0x30, 0x59, 0xe1, 0xb4, 0xfa, 0xbc, 0xa4, + 0xe2, 0x55, 0xb9, 0x79, 0x23, 0xf8, 0x73, 0xca, 0x75, 0xdb, 0xc0, 0x28, 0x0d, 0xb0, 0xa5, 0x11, + 0xd2, 0xdd, 0x77, 0x34, 0x82, 0xe5, 0x29, 0x14, 0x85, 0xe9, 0xb5, 0xf5, 0xa6, 0x2c, 0xdd, 0xfc, + 0x38, 0x18, 0x85, 0xac, 0x2a, 0xa5, 0xfa, 0x46, 0x7d, 0x63, 0x45, 0xdd, 0x28, 0xad, 0xd7, 0x9a, + 0xf2, 0x14, 0xca, 0xc2, 0xfc, 0x47, 0xa5, 0xfa, 0xb6, 0x08, 0x4b, 0xaa, 0xf5, 0x8d, 0xed, 0x9a, + 0x72, 0xaf, 0x74, 0x47, 0x96, 0xd0, 0x65, 0x40, 0xca, 0x66, 0x65, 0xad, 0x59, 0x2d, 0xab, 0x95, + 0xcd, 0xf5, 0xad, 0x52, 0x65, 0xbb, 0xbe, 0xb9, 0x21, 0x87, 0x50, 0x0c, 0xc2, 0xd5, 0xcd, 0x8d, + 0x9a, 0x0c, 0x37, 0x7f, 0x12, 0x86, 0x30, 0x0b, 0x80, 0xbc, 0x0c, 0x89, 0x9d, 0x8d, 0xe6, 0x56, + 0xad, 0x52, 0x5f, 0xae, 0xd7, 0xaa, 0xf2, 0x54, 0x6e, 0xee, 0xf1, 0x93, 0x85, 0x59, 0x9a, 0xb5, + 0x63, 0x91, 0x2e, 0xd6, 0x19, 0xb7, 0xa2, 0x1c, 0x44, 0xca, 0xa5, 0xca, 0xda, 0xce, 0x96, 0x2c, + 0xe5, 0xd2, 0x8f, 0x9f, 0x2c, 0x00, 0x2d, 0xc0, 0x79, 0x11, 0xbd, 0xc8, 0x03, 0x24, 0x9b, 0x4a, + 0x4d, 0x0e, 0xe5, 0x66, 0x1f, 0x3f, 0x59, 0x48, 0xb0, 0xb8, 0x8b, 0xe0, 0xb6, 0x57, 0x21, 0xd5, + 0xac, 0xac, 0xd6, 0xd6, 0x4b, 0x6a, 0x65, 0xb5, 0xb4, 0xb1, 0x52, 0x93, 0xa7, 0x73, 0xf3, 0x8f, + 0x9f, 0x2c, 0xc8, 0xa3, 0xfb, 0x83, 0x36, 0x51, 0x5f, 0xdf, 0xda, 0x54, 0xb6, 0xe5, 0xf0, 0xa0, + 0x09, 0x4e, 0x4b, 0xa8, 0x00, 0xc0, 0x6b, 0x2f, 0xd7, 0x6a, 0x55, 0x79, 0x26, 0x87, 0x1e, 0x3f, + 0x59, 0x48, 0xd3, 0xfc, 0x01, 0xdb, 0xa0, 0x6b, 0x90, 0xac, 0x28, 0xb5, 0xd2, 0x76, 0x4d, 0x6d, + 0x6e, 0x97, 0xb6, 0x9b, 0x72, 0x64, 0x30, 0x92, 0x00, 0x83, 0xa0, 0x22, 0x64, 0x4a, 0x3b, 0xdb, + 0x9b, 0xea, 0x50, 0xd9, 0x68, 0xee, 0xca, 0xe3, 0x27, 0x0b, 0x73, 0xb4, 0x6c, 0xa9, 0xe7, 0xda, + 0xc1, 0xf2, 0xaf, 0x83, 0x3c, 0xd4, 0x7f, 0x75, 0xa5, 0x22, 0xc7, 0x72, 0x97, 0x1f, 0x3f, 0x59, + 0x40, 0xa3, 0x43, 0x58, 0xa9, 0xa0, 0x6f, 0xc1, 0xe5, 0xed, 0x4f, 0xb6, 0x6a, 0xd5, 0x5a, 0xb3, + 0xa2, 0x0e, 0x0f, 0x3b, 0x9e, 0xcb, 0x3e, 0x7e, 0xb2, 0x30, 0x4f, 0xeb, 0x8c, 0x0d, 0xfd, 0x0d, + 0x90, 0x9b, 0xdb, 0x4a, 0xad, 0xb4, 0xae, 0xd6, 0x37, 0x56, 0x6a, 0x4d, 0xb6, 0x58, 0x30, 0xe8, + 0xd2, 0xc8, 0x5e, 0xa7, 0x43, 0xd8, 0xa8, 0x7d, 0x34, 0x82, 0x9f, 0x18, 0x94, 0x1f, 0xd9, 0xbe, + 0x68, 0x01, 0xe2, 0xeb, 0xf5, 0x15, 0xa5, 0xc4, 0x70, 0x93, 0xb9, 0xcc, 0xe3, 0x27, 0x0b, 0x29, + 0x5a, 0xce, 0xdf, 0x8c, 0xb9, 0xd8, 0x8f, 0x7f, 0x76, 0x75, 0xea, 0x2f, 0x7e, 0x7e, 0x75, 0xaa, + 0x7c, 0xfd, 0xf3, 0xff, 0xb8, 0x3a, 0xf5, 0xf9, 0xd1, 0x55, 0xe9, 0xd7, 0x47, 0x57, 0xa5, 0x2f, + 0x8e, 0xae, 0x4a, 0xff, 0x7e, 0x74, 0x55, 0xfa, 0xa3, 0x2f, 0xaf, 0x4e, 0xfd, 0xfa, 0xcb, 0xab, + 0x53, 0x5f, 0x7c, 0x79, 0x75, 0xea, 0xd3, 0x08, 0xd7, 0xeb, 0x56, 0x84, 0x9d, 0x1c, 0xdf, 0xfa, + 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x4d, 0x79, 0xf4, 0x54, 0xff, 0x3e, 0x00, 0x00, } func (this *Lease) Equal(that interface{}) bool { @@ -5645,13 +5645,6 @@ func (m *Payload) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.TraceID != 0 { - i = encodeVarintJobs(dAtA, i, uint64(m.TraceID)) - i-- - dAtA[i] = 0x1 - i-- - dAtA[i] = 0xd0 - } if m.Details != nil { { size := m.Details.Size() @@ -6055,6 +6048,13 @@ func (m *Progress) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.TraceID != 0 { + i = encodeVarintJobs(dAtA, i, uint64(m.TraceID)) + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0xa8 + } if m.Details != nil { { size := m.Details.Size() @@ -7515,9 +7515,6 @@ func (m *Payload) Size() (n int) { if m.Noncancelable { n += 3 } - if m.TraceID != 0 { - n += 2 + sovJobs(uint64(m.TraceID)) - } return n } @@ -7672,6 +7669,9 @@ func (m *Progress) Size() (n int) { if m.Details != nil { n += m.Details.Size() } + if m.TraceID != 0 { + n += 2 + sovJobs(uint64(m.TraceID)) + } return n } @@ -15983,25 +15983,6 @@ func (m *Payload) Unmarshal(dAtA []byte) error { } m.Details = &Payload_Migration{v} iNdEx = postIndex - case 26: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field TraceID", wireType) - } - m.TraceID = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowJobs - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.TraceID |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } default: iNdEx = preIndex skippy, err := skipJobs(dAtA[iNdEx:]) @@ -16534,6 +16515,25 @@ func (m *Progress) Unmarshal(dAtA []byte) error { } m.Details = &Progress_Migration{v} iNdEx = postIndex + case 21: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TraceID", wireType) + } + m.TraceID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowJobs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TraceID |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipJobs(dAtA[iNdEx:]) diff --git a/pkg/jobs/jobspb/jobs.proto b/pkg/jobs/jobspb/jobs.proto index a61407338405..a4611faaf318 100644 --- a/pkg/jobs/jobspb/jobs.proto +++ b/pkg/jobs/jobspb/jobs.proto @@ -699,7 +699,7 @@ message Payload { NewSchemaChangeDetails newSchemaChange = 24; MigrationDetails migration = 25; } - uint64 trace_id = 26 [(gogoproto.customname) = "TraceID"]; + reserved 26; // NEXT ID: 27. } @@ -725,6 +725,8 @@ message Progress { NewSchemaChangeProgress newSchemaChange = 19; MigrationProgress migration = 20; } + + uint64 trace_id = 21 [(gogoproto.customname) = "TraceID"]; } enum Type { diff --git a/pkg/sql/crdb_internal.go b/pkg/sql/crdb_internal.go index 53e79e69b282..015e3e796aa6 100644 --- a/pkg/sql/crdb_internal.go +++ b/pkg/sql/crdb_internal.go @@ -728,7 +728,6 @@ CREATE TABLE crdb_internal.jobs ( leaseNode = tree.NewDInt(tree.DInt(payload.Lease.NodeID)) } errorStr = tree.NewDString(payload.Error) - traceID = tree.NewDInt(tree.DInt(payload.TraceID)) } // Extract data from the progress field. @@ -763,6 +762,7 @@ CREATE TABLE crdb_internal.jobs ( } } } + traceID = tree.NewDInt(tree.DInt(progress.TraceID)) } } From 0f6722ebc473fe0ed57f2a707087d3d2015e739d Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Mon, 17 May 2021 16:03:44 +0200 Subject: [PATCH 2/3] kvserver: document pitfalls around DisableRaftSnapshotQueue Disabling this might be tempting sometimes, but it makes up-replication flaky due to corner cases in which we need a Raft snapshot to unwedge things. I just spent a lot of time on this, so document what I learned I had (hackily) implemented the various improvements mentioned in this comment, but they're too dodgy for production code, and besides there would still be more holes left that would be more difficult to plug. Release note: None --- pkg/kv/kvserver/testing_knobs.go | 51 +++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/pkg/kv/kvserver/testing_knobs.go b/pkg/kv/kvserver/testing_knobs.go index 811aa512980e..c110d497b537 100644 --- a/pkg/kv/kvserver/testing_knobs.go +++ b/pkg/kv/kvserver/testing_knobs.go @@ -116,7 +116,56 @@ type StoreTestingKnobs struct { // DisableTimeSeriesMaintenanceQueue disables the time series maintenance // queue. DisableTimeSeriesMaintenanceQueue bool - // DisableRaftSnapshotQueue disables the raft snapshot queue. + // DisableRaftSnapshotQueue disables the raft snapshot queue. Use this + // sparingly, as it tends to produce flaky tests during up-replication where + // the explicit learner snapshot gets lost. Since uninitialized replicas + // reject the raft leader's MsgApp with a rejection that hints at index zero, + // Raft will always want to send another snapshot if it gets to talk to the + // follower before it applied the snapshot. In the common case, the mechanism + // we have to suppress raft snapshots while an explicit snapshot is in flight + // avoids the duplication of work. However, in the uncommon case, the raft + // leader is on a different store and so no suppression takes place. This + // duplication is avoided by turning the raft snapshot queue off, but + // unfortunately there are certain conditions under which the explicit + // snapshot may fail to produce the desired result of an initialized follower + // which the leader will append to. + // + // For example, if there is a term change between when the snapshot is + // constructed and received, the follower will silently drop the snapshot + // without signaling an error to the sender, i.e. the leader. (To work around + // that, once could adjust the term for the message in that case in + // `(*Replica).stepRaftGroup` to prevent the message from getting dropped). + // + // Another problem is that the snapshot may apply but fail to inform the + // leader of this fact. Once a Raft leader has determined that a follower + // needs a snapshot, it will keep asking for a snapshot to be sent until a) a + // call to .ReportSnapshotStatus acks that a snapshot got applied, or b) an + // MsgAppResp is received from from the follower acknowledging an index + // *greater than or equal to the PendingSnapshotIndex* it tracks for that + // follower (such an MsgAppResp is emitted in response to applying the + // snapshot), whichever occurs first. + // + // + // However, neither may happen despite the snapshot applying successfully. The + // call to `ReportSnapshotStatus` may not occur on the correct Replica, since + // Raft leader can change during replication operations, and MsgAppResp can + // get dropped. Even if the MsgAppResp does get to the leader, the contained + // index (which is the index of the snapshot that got applied) may be less + // than the index at which the leader asked for a snapshot (since the applied + // snapshot is an external snapshot, and may have been initiated before the + // raft leader even wanted a snapshot) - again leaving the leader to continue + // asking for a snapshot. Lastly, even if we improved the raft code to accept + // all snapshots that allow it to continue replicating the log, there may have + // been a log truncation separating the snapshot from the log. + // + // Either way, the result in all of the rare cases above this is that an + // additional snapshot must be sent from the Raft snapshot queue until the + // leader will include the follower in log replication. It is thus ill-advised + // to turn the snap queue off except when this is known to be a good idea. + // + // An example of a test that becomes flaky with this set to false is + // TestReportUnreachableRemoveRace, though it needs a 20-node roachprod-stress + // cluster to reliably reproduce this within a few minutes. DisableRaftSnapshotQueue bool // DisableConsistencyQueue disables the consistency checker. DisableConsistencyQueue bool From 6b9168de33628257f3f1fdb17af1f8205ee21d32 Mon Sep 17 00:00:00 2001 From: Tobias Grieger Date: Mon, 29 Mar 2021 12:38:45 +0200 Subject: [PATCH 3/3] kvserver: simplify and document handling of ReplicaPlaceholder This commit cleans up the handling of ReplicaPlaceholder and documents their semantics better. I embarked on this while looking into an unrelated test flake. It occurred to me that we were not installing placeholders before the snapshot transfer, but only after. This seems like a bad idea now that our average snapshot size has gone up, as placeholders help avoid duplicate work; we now install them before accepting the range data. In practice, we currently only allow once snapshot inflight per store, so it is not clear that that improvement is buying us anything. I think that the improved clarity in this commit will, though. Concretely, we now have two simple rules for placeholders: 1. they only exist for uninitialized replicas (and must exist for those, or we can end up with overlapping replicas) 2. you write it --> you (and *only* you) remove it. 1. was true before but the documentation was not clear. 2. was not true, as there were a few out-of-band places that removed placeholders (and didn't clearly explain why - sounded like we were previously leaking placeholders). There's also a bit of extra cleanup that clarifies things - for example, since caller to `applySnapshot` already checked for an empty snapshot, the placeholder removal within it effectively dead code and is now removed. Also, in receiveSnapshot, there was an optimistic check that allowed us to discard snapshots early. It was always confusing to keep track of which one is which and now that we've lifted the authoritative check to the top of receiveSnapshot, we got to delete the other one. I ended up significantly rewriting TestReportUnreachableRemoveRace (as it is much more flaky with the early-installed placeholders), and so this also: Closes #62955 as verified with 20k+ iterations of roachprod-stress on a 20 node 8cpu cluster. As part of this I went ahead and made replication changes carried out by TestCluster aware of retriable replication errors, which it will now retry for up to 45s. Release note: None --- pkg/kv/kvserver/client_raft_test.go | 156 ++++++++++++------ pkg/kv/kvserver/client_replica_test.go | 11 +- pkg/kv/kvserver/client_split_test.go | 8 +- .../replica_application_state_machine.go | 2 +- pkg/kv/kvserver/replica_learner_test.go | 11 +- pkg/kv/kvserver/replica_placeholder.go | 52 +++++- pkg/kv/kvserver/replica_raft.go | 128 +++++++++----- pkg/kv/kvserver/replica_raftstorage.go | 45 ++--- pkg/kv/kvserver/replicate_queue_test.go | 1 + pkg/kv/kvserver/store.go | 12 +- pkg/kv/kvserver/store_raft.go | 103 +++++------- pkg/kv/kvserver/store_remove_replica.go | 126 +++++++++++--- pkg/kv/kvserver/store_snapshot.go | 91 +++++----- pkg/kv/kvserver/store_test.go | 105 +++++++++--- pkg/testutils/testcluster/testcluster.go | 43 +++-- 15 files changed, 580 insertions(+), 314 deletions(-) diff --git a/pkg/kv/kvserver/client_raft_test.go b/pkg/kv/kvserver/client_raft_test.go index 3815ce0fd168..8a5858f4fd4a 100644 --- a/pkg/kv/kvserver/client_raft_test.go +++ b/pkg/kv/kvserver/client_raft_test.go @@ -50,6 +50,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/leaktest" "github.com/cockroachdb/cockroach/pkg/util/log" "github.com/cockroachdb/cockroach/pkg/util/protoutil" + "github.com/cockroachdb/cockroach/pkg/util/randutil" "github.com/cockroachdb/cockroach/pkg/util/stop" "github.com/cockroachdb/cockroach/pkg/util/timeutil" "github.com/cockroachdb/cockroach/pkg/util/uuid" @@ -1499,7 +1500,12 @@ func TestReplicateAfterRemoveAndSplit(t *testing.T) { tc.AddAndStartServer(t, stickyServerArgs[2]) // Try to up-replicate the RHS of the split to store 2. - if _, err := tc.AddVoters(splitKey, tc.Target(3)); !testutils.IsError(err, kvserver.IntersectingSnapshotMsg) { + // Don't use tc.AddVoter because we expect a retriable error and want it + // returned to us. + if _, err := tc.Servers[0].DB().AdminChangeReplicas( + ctx, splitKey, tc.LookupRangeOrFatal(t, splitKey), + roachpb.MakeReplicationChanges(roachpb.ADD_VOTER, tc.Target(3)), + ); !kvserver.IsRetriableReplicationChangeError(err) { t.Fatalf("unexpected error %v", err) } @@ -1507,6 +1513,7 @@ func TestReplicateAfterRemoveAndSplit(t *testing.T) { // to store 2 will cause the obsolete replica to be GC'd allowing a // subsequent replication to succeed. tc.GetFirstStoreFromServer(t, 3).SetReplicaGCQueueActive(true) + tc.AddVotersOrFatal(t, splitKey, tc.Target(3)) } // Test that when a Raft group is not able to establish a quorum, its Raft log @@ -2492,6 +2499,9 @@ func TestReportUnreachableRemoveRace(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + rng, seed := randutil.NewTestPseudoRand() + t.Logf("seed is %d", seed) + ctx := context.Background() tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ @@ -2502,54 +2512,78 @@ func TestReportUnreachableRemoveRace(t *testing.T) { tc.AddVotersOrFatal(t, key, tc.Targets(1, 2)...) require.NoError(t, tc.WaitForVoters(key, tc.Targets(1, 2)...)) -outer: for i := 0; i < 5; i++ { - for leaderIdx := range tc.Servers { - repl := tc.GetFirstStoreFromServer(t, leaderIdx).LookupReplica(roachpb.RKey(key)) - require.NotNil(t, repl) - if repl.RaftStatus().SoftState.RaftState == raft.StateLeader { - for replicaIdx := range tc.Servers { - if replicaIdx == leaderIdx { - continue - } - repDesc, err := repl.GetReplicaDescriptor() - if err != nil { - t.Fatal(err) - } - if lease, _ := repl.GetLease(); lease.Replica.Equal(repDesc) { - tc.TransferRangeLeaseOrFatal(t, *repl.Desc(), tc.Target(replicaIdx)) - } - tc.RemoveVotersOrFatal(t, key, tc.Target(leaderIdx)) - // We want to stop all nodes from talking to the replicaIdx, so need - // to trip the breaker on all servers but it. - for i := range tc.Servers { - if i != replicaIdx { - cb := tc.Servers[i].RaftTransport().GetCircuitBreaker(tc.Target(replicaIdx).NodeID, rpc.DefaultClass) - cb.Break() - } - } - time.Sleep(tc.GetFirstStoreFromServer(t, replicaIdx).GetStoreConfig().CoalescedHeartbeatsInterval) - for i := range tc.Servers { - if i != replicaIdx { - cb := tc.Servers[i].RaftTransport().GetCircuitBreaker(tc.Target(replicaIdx).NodeID, rpc.DefaultClass) - cb.Reset() - } - } - // Make sure the old replica was actually removed, before we try to re-adding it. - testutils.SucceedsSoon(t, func() error { - if oldRepl := tc.GetFirstStoreFromServer(t, leaderIdx).LookupReplica(roachpb.RKey(key)); oldRepl != nil { - return errors.Errorf("Expected replica %s to be removed", oldRepl) - } - return nil - }) - tc.AddVotersOrFatal(t, key, tc.Target(leaderIdx)) - require.NoError(t, tc.WaitForVoters(key, tc.Target(leaderIdx))) - continue outer + // Find the Raft leader. + var leaderIdx int + var leaderRepl *kvserver.Replica + testutils.SucceedsSoon(t, func() error { + for idx := range tc.Servers { + repl := tc.GetFirstStoreFromServer(t, idx).LookupReplica(roachpb.RKey(key)) + require.NotNil(t, repl) + if repl.RaftStatus().SoftState.RaftState == raft.StateLeader { + leaderIdx = idx + leaderRepl = repl + return nil } - t.Fatal("could not find raft replica") } + return errors.New("no Raft leader found") + }) + + // Raft leader found. Make sure it doesn't have the lease (transferring it + // away if necessary, which entails picking a random other node and sending + // the lease to it). We'll also partition this random node away from the rest + // as this was (way back) how we triggered problems with coalesced heartbeats. + partitionedMaybeLeaseholderIdx := (leaderIdx + 1 + rng.Intn(tc.NumServers()-1)) % tc.NumServers() + t.Logf("leader is idx=%d, partitioning idx=%d", leaderIdx, partitionedMaybeLeaseholderIdx) + leaderRepDesc, err := leaderRepl.GetReplicaDescriptor() + require.NoError(t, err) + if lease, _ := leaderRepl.GetLease(); lease.OwnedBy(leaderRepDesc.StoreID) { + tc.TransferRangeLeaseOrFatal(t, *leaderRepl.Desc(), tc.Target(partitionedMaybeLeaseholderIdx)) } - i-- // try again + + // Remove the raft leader. + t.Logf("removing leader") + tc.RemoveVotersOrFatal(t, key, tc.Target(leaderIdx)) + + // Pseudo-partition partitionedMaybeLeaseholderIdx away from everyone else. We do this by tripping + // the circuit breaker on all other nodes. + t.Logf("partitioning") + for i := range tc.Servers { + if i != partitionedMaybeLeaseholderIdx { + cb := tc.Servers[i].RaftTransport().GetCircuitBreaker(tc.Target(partitionedMaybeLeaseholderIdx).NodeID, rpc.DefaultClass) + cb.Break() + } + } + + // Wait out the heartbeat interval and resolve the partition. + heartbeatInterval := tc.GetFirstStoreFromServer(t, partitionedMaybeLeaseholderIdx).GetStoreConfig().CoalescedHeartbeatsInterval + time.Sleep(heartbeatInterval) + t.Logf("resolving partition") + for i := range tc.Servers { + if i != partitionedMaybeLeaseholderIdx { + cb := tc.Servers[i].RaftTransport().GetCircuitBreaker(tc.Target(partitionedMaybeLeaseholderIdx).NodeID, rpc.DefaultClass) + cb.Reset() + } + } + + t.Logf("waiting for replicaGC of removed leader replica") + // Make sure the old replica was actually removed by replicaGC, before we + // try to re-add it. Otherwise the addition below might fail. One shot here + // is often enough, but not always; in the worst case we need to wait out + // something on the order of a election timeout plus + // ReplicaGCQueueSuspectTimeout before replicaGC will be attempted (and will + // then succeed on the first try). + testutils.SucceedsSoon(t, func() error { + s := tc.GetFirstStoreFromServer(t, leaderIdx) + s.MustForceReplicaGCScanAndProcess() + if oldRepl := tc.GetFirstStoreFromServer(t, leaderIdx).LookupReplica(roachpb.RKey(key)); oldRepl != nil { + return errors.Errorf("Expected replica %s to be removed", oldRepl) + } + return nil + }) + t.Logf("re-adding leader") + tc.AddVotersOrFatal(t, key, tc.Target(leaderIdx)) + require.NoError(t, tc.WaitForVoters(key, tc.Target(leaderIdx))) } } @@ -4873,6 +4907,7 @@ func TestProcessSplitAfterRightHandSideHasBeenRemoved(t *testing.T) { require.NoError(t, db.Run(ctx, b)) } ensureNoTombstone := func(t *testing.T, store *kvserver.Store, rangeID roachpb.RangeID) { + t.Helper() var tombstone roachpb.RangeTombstone tombstoneKey := keys.RangeTombstoneKey(rangeID) ok, err := storage.MVCCGetProto( @@ -5040,7 +5075,10 @@ func TestProcessSplitAfterRightHandSideHasBeenRemoved(t *testing.T) { // Unsuccessful because the RHS will not accept the learner snapshot // and will be rolled back. Nevertheless it will have learned that it // has been removed at the old replica ID. - _, err = tc.AddVoters(keyB, tc.Target(0)) + _, err = tc.Servers[0].DB().AdminChangeReplicas( + ctx, keyB, tc.LookupRangeOrFatal(t, keyB), + roachpb.MakeReplicationChanges(roachpb.ADD_VOTER, tc.Target(0)), + ) require.True(t, kvserver.IsRetriableReplicationChangeError(err), err) // Without a partitioned RHS we'll end up always writing a tombstone here because @@ -5090,7 +5128,10 @@ func TestProcessSplitAfterRightHandSideHasBeenRemoved(t *testing.T) { // Unsuccessfuly because the RHS will not accept the learner snapshot // and will be rolled back. Nevertheless it will have learned that it // has been removed at the old replica ID. - _, err = tc.AddVoters(keyB, tc.Target(0)) + _, err = tc.Servers[0].DB().AdminChangeReplicas( + ctx, keyB, tc.LookupRangeOrFatal(t, keyB), + roachpb.MakeReplicationChanges(roachpb.ADD_VOTER, tc.Target(0)), + ) require.True(t, kvserver.IsRetriableReplicationChangeError(err), err) // Without a partitioned RHS we'll end up always writing a tombstone here because @@ -5158,10 +5199,15 @@ func TestProcessSplitAfterRightHandSideHasBeenRemoved(t *testing.T) { // Remove and re-add the RHS to create a new uninitialized replica at // a higher replica ID. This will lead to a tombstone being written. tc.RemoveVotersOrFatal(t, keyB, tc.Target(0)) - // Unsuccessful because the RHS will not accept the learner snapshot - // and will be rolled back. Nevertheless it will have learned that it - // has been removed at the old replica ID. - _, err = tc.AddVoters(keyB, tc.Target(0)) + // Unsuccessful because the RHS will not accept the learner snapshot and + // will be rolled back. Nevertheless it will have learned that it has been + // removed at the old replica ID. We don't use tc.AddVoters because that + // will retry until it runs out of time, since we're creating a + // retriable-looking situation here that will persist. + _, err = tc.Servers[0].DB().AdminChangeReplicas( + ctx, keyB, tc.LookupRangeOrFatal(t, keyB), + roachpb.MakeReplicationChanges(roachpb.ADD_VOTER, tc.Target(0)), + ) require.True(t, kvserver.IsRetriableReplicationChangeError(err), err) // Ensure that the replica exists with the higher replica ID. repl, err := tc.GetFirstStoreFromServer(t, 0).GetReplica(rhsInfo.Desc.RangeID) @@ -5219,7 +5265,13 @@ func TestProcessSplitAfterRightHandSideHasBeenRemoved(t *testing.T) { // Unsuccessfuly because the RHS will not accept the learner snapshot // and will be rolled back. Nevertheless it will have learned that it // has been removed at the old replica ID. - _, err = tc.AddVoters(keyB, tc.Target(0)) + // + // Not using tc.AddVoters because we expect an error, but that error + // would be retried internally. + _, err = tc.Servers[0].DB().AdminChangeReplicas( + ctx, keyB, tc.LookupRangeOrFatal(t, keyB), + roachpb.MakeReplicationChanges(roachpb.ADD_VOTER, tc.Target(0)), + ) require.True(t, kvserver.IsRetriableReplicationChangeError(err), err) // Ensure that there's no tombstone. // The RHS on store 0 never should have heard about its original ID. diff --git a/pkg/kv/kvserver/client_replica_test.go b/pkg/kv/kvserver/client_replica_test.go index 2701fe27fe56..58da6d76bcdc 100644 --- a/pkg/kv/kvserver/client_replica_test.go +++ b/pkg/kv/kvserver/client_replica_test.go @@ -2628,7 +2628,12 @@ func TestReplicaTombstone(t *testing.T) { // this as a heartbeat. This demonstrates case (4) where a raft message // to a newer replica ID (in this case a heartbeat) removes an initialized // Replica. - _, err = tc.AddVoters(key, tc.Target(2)) + // + // Don't use tc.AddVoter; this would retry internally as we're faking a + // a snapshot error here (and these are all considered retriable). + _, err = tc.Servers[0].DB().AdminChangeReplicas( + ctx, key, tc.LookupRangeOrFatal(t, key), roachpb.MakeReplicationChanges(roachpb.ADD_VOTER, tc.Target(2)), + ) require.Regexp(t, "boom", err) tombstone := waitForTombstone(t, store.Engine(), rangeID) require.Equal(t, roachpb.ReplicaID(4), tombstone.NextReplicaID) @@ -2644,7 +2649,9 @@ func TestReplicaTombstone(t *testing.T) { // We could replica GC these replicas without too much extra work but they // also should be rare. Note this is not new with learner replicas. setMinHeartbeat(5) - _, err = tc.AddVoters(key, tc.Target(2)) + _, err = tc.Servers[0].DB().AdminChangeReplicas( + ctx, key, tc.LookupRangeOrFatal(t, key), roachpb.MakeReplicationChanges(roachpb.ADD_VOTER, tc.Target(2)), + ) require.Regexp(t, "boom", err) // We will start out reading the old tombstone so keep retrying. testutils.SucceedsSoon(t, func() error { diff --git a/pkg/kv/kvserver/client_split_test.go b/pkg/kv/kvserver/client_split_test.go index fb409d3f35bf..332719247a74 100644 --- a/pkg/kv/kvserver/client_split_test.go +++ b/pkg/kv/kvserver/client_split_test.go @@ -3308,7 +3308,9 @@ func TestSplitTriggerMeetsUnexpectedReplicaID(t *testing.T) { // second node). g := ctxgroup.WithContext(ctx) g.GoCtx(func(ctx context.Context) error { - _, err := tc.AddVoters(k, tc.Target(1)) + _, err := tc.Servers[0].DB().AdminChangeReplicas( + ctx, k, tc.LookupRangeOrFatal(t, k), roachpb.MakeReplicationChanges(roachpb.ADD_VOTER, tc.Target(1)), + ) return err }) @@ -3343,7 +3345,9 @@ func TestSplitTriggerMeetsUnexpectedReplicaID(t *testing.T) { // Now repeatedly re-add the learner on the rhs, so it has a // different replicaID than the split trigger expects. add := func() { - _, err := tc.AddVoters(kRHS, tc.Target(1)) + _, err := tc.Servers[0].DB().AdminChangeReplicas( + ctx, kRHS, tc.LookupRangeOrFatal(t, kRHS), roachpb.MakeReplicationChanges(roachpb.ADD_VOTER, tc.Target(1)), + ) // The "snapshot intersects existing range" error is expected if the store // has not heard a raft message addressed to a later replica ID while the // "was not found on" error is expected if the store has heard that it has diff --git a/pkg/kv/kvserver/replica_application_state_machine.go b/pkg/kv/kvserver/replica_application_state_machine.go index 05d498916e8a..37dde422911c 100644 --- a/pkg/kv/kvserver/replica_application_state_machine.go +++ b/pkg/kv/kvserver/replica_application_state_machine.go @@ -73,7 +73,7 @@ type nonDeterministicFailure struct { // The provided format string should be safe for reporting. func makeNonDeterministicFailure(format string, args ...interface{}) error { - err := errors.Newf(format, args...) + err := errors.AssertionFailedWithDepthf(1, format, args...) return &nonDeterministicFailure{ wrapped: err, safeExpl: err.Error(), diff --git a/pkg/kv/kvserver/replica_learner_test.go b/pkg/kv/kvserver/replica_learner_test.go index 6cc6121ebded..089fa0bf1387 100644 --- a/pkg/kv/kvserver/replica_learner_test.go +++ b/pkg/kv/kvserver/replica_learner_test.go @@ -669,7 +669,16 @@ func TestLearnerAdminChangeReplicasRace(t *testing.T) { scratchStartKey := tc.ScratchRange(t) g := ctxgroup.WithContext(ctx) g.GoCtx(func(ctx context.Context) error { - _, err := tc.AddVoters(scratchStartKey, tc.Target(1)) + // NB: we don't use tc.AddVoters because that will auto-retry + // and the test expects to see the error that results on the + // first attempt. + desc, err := tc.LookupRange(scratchStartKey) + if err != nil { + return err + } + _, err = tc.Servers[0].DB().AdminChangeReplicas( + ctx, scratchStartKey, desc, roachpb.MakeReplicationChanges(roachpb.ADD_VOTER, tc.Target(1)), + ) return err }) diff --git a/pkg/kv/kvserver/replica_placeholder.go b/pkg/kv/kvserver/replica_placeholder.go index 26a23f9cacc9..9602082cd0c1 100644 --- a/pkg/kv/kvserver/replica_placeholder.go +++ b/pkg/kv/kvserver/replica_placeholder.go @@ -12,16 +12,16 @@ package kvserver import ( "fmt" + "sync/atomic" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/google/btree" ) // ReplicaPlaceholder represents a "lock" of a part of the keyspace on a given -// *Store for the application of a (preemptive or Raft) snapshot. Placeholders +// *Store for the receipt and application of a raft snapshot. Placeholders // are kept synchronously in two places in (*Store).mu, namely the -// replicaPlaceholders and replicaByKey maps, and exist only while the Raft -// scheduler tries to apply raft.Ready containing a snapshot to some Replica. +// replicaPlaceholders and replicaByKey maps. // // To see why placeholders are necessary, consider the case in which two // snapshots arrive at a Store, one for r1 and bounds [a,c) and the other for r2 @@ -52,12 +52,42 @@ import ( // existing placeholder (inserting its own atomically when none found), so that // g2 would later fail the overlap check on g1's placeholder. // -// Placeholders are removed by the goroutine that inserted them at the end of -// the respective Raft cycle, so they usually live only for as long as it takes -// to write the snapshot to disk. See (*Store).processRaftSnapshotRequest for -// details. +// The rules for placeholders are as follows: +// +// - placeholders are only installed for uninitialized replicas (under raftMu). +// In particular, a snapshot that gets sent to an initialized replica installs +// no placeholder (the initialized replica plays the role of the placeholder). +// - they do not overlap any initialized replica's key bounds. (This invariant +// is maintained via Store.mu.replicasByKey). +// - a placeholder can only be removed by the operation that installed it, and +// that operation *must* eventually remove it. In practice, they are inserted +// before receiving the snapshot data, so they are fairly long-lived. They +// are removed when the receipt of the snapshot fails, the snapshot is discarded, +// or the snapshot was fully applied (in which case the placeholder is exchanged +// for a RangeDescriptor). +// - placeholders must not be copied (i.e. always pass by reference). +// +// In particular, when removing a placeholder we don't have to worry about +// whether we're removing our own or someone else's. This is because they +// can't overlap; and if we inserted one it's still there (since nobody else +// can delete it). +// +// Note also that both initialized and uninitialized replicas can get +// replicaGC'ed (which also removes any placeholders). It is thus possible for a +// snapshot to begin streaming to either an uninitialized (via a placeholder) or +// initialized replica, and for that replica to be removed by replicaGC by the +// time the snapshot is ready to apply. This will simply result the snapshot +// being discarded (and has to - applying the snapshot could lead to the races +// that placeholders were introduced for in the first place) due to the +// replicaID being rejected by the range tombstone (written during replicaGC) in +// snapshot application. +// +// See (*Store).receiveSnapshot for where placeholders are installed. This will +// eventually call (*Store).processRaftSnapshotRequest which triggers the actual +// application. type ReplicaPlaceholder struct { rangeDesc roachpb.RangeDescriptor + tainted int32 // atomic } var _ rangeKeyItem = (*ReplicaPlaceholder)(nil) @@ -77,6 +107,10 @@ func (r *ReplicaPlaceholder) Less(i btree.Item) bool { } func (r *ReplicaPlaceholder) String() string { - return fmt.Sprintf("range=%d [%s-%s) (placeholder)", - r.Desc().RangeID, r.rangeDesc.StartKey, r.rangeDesc.EndKey) + tainted := "" + if atomic.LoadInt32(&r.tainted) != 0 { + tainted = ",tainted" + } + return fmt.Sprintf("range=%d [%s-%s) (placeholder%s)", + r.Desc().RangeID, r.rangeDesc.StartKey, r.rangeDesc.EndKey, tainted) } diff --git a/pkg/kv/kvserver/replica_raft.go b/pkg/kv/kvserver/replica_raft.go index aac31229ff18..d486fa43b471 100644 --- a/pkg/kv/kvserver/replica_raft.go +++ b/pkg/kv/kvserver/replica_raft.go @@ -420,6 +420,21 @@ func (r *Replica) stepRaftGroup(req *RaftMessageRequest) error { // we expect the originator to campaign instead. r.unquiesceWithOptionsLocked(false /* campaignOnWake */) r.mu.lastUpdateTimes.update(req.FromReplica.ReplicaID, timeutil.Now()) + if req.Message.Type == raftpb.MsgSnap { + // Occasionally a snapshot message may arrive under an outdated term, + // which would lead to Raft discarding the snapshot. This should be + // really rare in practice, but it does happen in tests and in particular + // can happen to the synchronous snapshots on the learner path, which + // will then have to wait for the raft snapshot queue to send another + // snapshot. However, in some tests it is desirable to disable the + // raft snapshot queue. This workaround makes that possible. + // + // See TestReportUnreachableRemoveRace for the test that prompted + // this addition. + if term := raftGroup.BasicStatus().Term; term > req.Message.Term { + req.Message.Term = term + } + } err := raftGroup.Step(req.Message) if errors.Is(err, raft.ErrProposalDropped) { // A proposal was forwarded to this replica but we couldn't propose it. @@ -444,8 +459,14 @@ func (r *Replica) raftSchedulerCtx(schedulerCtx context.Context) context.Context return schedulerCtx } +type handleSnapshotStats struct { + offered bool + applied bool +} + type handleRaftReadyStats struct { applyCommittedEntriesStats + snap handleSnapshotStats } // noSnap can be passed to handleRaftReady when no snapshot should be processed. @@ -473,8 +494,11 @@ func (r *Replica) handleRaftReady( // non-sensitive cue as to what happened. func (r *Replica) handleRaftReadyRaftMuLocked( ctx context.Context, inSnap IncomingSnapshot, -) (handleRaftReadyStats, string, error) { +) (_ handleRaftReadyStats, _ string, foo error) { var stats handleRaftReadyStats + if inSnap.State != nil { + stats.snap.offered = true + } var hasReady bool var rd raft.Ready @@ -549,51 +573,63 @@ func (r *Replica) handleRaftReadyRaftMuLocked( leaderID = roachpb.ReplicaID(rd.SoftState.Lead) } - if !raft.IsEmptySnap(rd.Snapshot) { - snapUUID, err := uuid.FromBytes(rd.Snapshot.Data) - if err != nil { - const expl = "invalid snapshot id" - return stats, expl, errors.Wrap(err, expl) - } - if inSnap.SnapUUID == (uuid.UUID{}) { - log.Fatalf(ctx, "programming error: a snapshot application was attempted outside of the streaming snapshot codepath") - } - if snapUUID != inSnap.SnapUUID { - log.Fatalf(ctx, "incoming snapshot id doesn't match raft snapshot id: %s != %s", snapUUID, inSnap.SnapUUID) - } - - // Applying this snapshot may require us to subsume one or more of our right - // neighbors. This occurs if this replica is informed about the merges via a - // Raft snapshot instead of a MsgApp containing the merge commits, e.g., - // because it went offline before the merge commits applied and did not come - // back online until after the merge commits were truncated away. - subsumedRepls, releaseMergeLock := r.maybeAcquireSnapshotMergeLock(ctx, inSnap) - defer releaseMergeLock() - - if err := r.applySnapshot(ctx, inSnap, rd.Snapshot, rd.HardState, subsumedRepls); err != nil { - const expl = "while applying snapshot" - return stats, expl, errors.Wrap(err, expl) - } + if inSnap.State != nil { + if !raft.IsEmptySnap(rd.Snapshot) { + snapUUID, err := uuid.FromBytes(rd.Snapshot.Data) + if err != nil { + const expl = "invalid snapshot id" + return stats, expl, errors.Wrap(err, expl) + } + if inSnap.SnapUUID == (uuid.UUID{}) { + log.Fatalf(ctx, "programming error: a snapshot application was attempted outside of the streaming snapshot codepath") + } + if snapUUID != inSnap.SnapUUID { + log.Fatalf(ctx, "incoming snapshot id doesn't match raft snapshot id: %s != %s", snapUUID, inSnap.SnapUUID) + } - // r.mu.lastIndex, r.mu.lastTerm and r.mu.raftLogSize were updated in - // applySnapshot, but we also want to make sure we reflect these changes in - // the local variables we're tracking here. - r.mu.RLock() - lastIndex = r.mu.lastIndex - lastTerm = r.mu.lastTerm - raftLogSize = r.mu.raftLogSize - r.mu.RUnlock() - - // We refresh pending commands after applying a snapshot because this - // replica may have been temporarily partitioned from the Raft group and - // missed leadership changes that occurred. Suppose node A is the leader, - // and then node C gets partitioned away from the others. Leadership passes - // back and forth between A and B during the partition, but when the - // partition is healed node A is leader again. - if !r.store.TestingKnobs().DisableRefreshReasonSnapshotApplied && - refreshReason == noReason { - refreshReason = reasonSnapshotApplied + // Applying this snapshot may require us to subsume one or more of our right + // neighbors. This occurs if this replica is informed about the merges via a + // Raft snapshot instead of a MsgApp containing the merge commits, e.g., + // because it went offline before the merge commits applied and did not come + // back online until after the merge commits were truncated away. + subsumedRepls, releaseMergeLock := r.maybeAcquireSnapshotMergeLock(ctx, inSnap) + defer releaseMergeLock() + + if err := r.applySnapshot(ctx, inSnap, rd.Snapshot, rd.HardState, subsumedRepls); err != nil { + const expl = "while applying snapshot" + return stats, expl, errors.Wrap(err, expl) + } + stats.snap.applied = true + + // r.mu.lastIndex, r.mu.lastTerm and r.mu.raftLogSize were updated in + // applySnapshot, but we also want to make sure we reflect these changes in + // the local variables we're tracking here. + r.mu.RLock() + lastIndex = r.mu.lastIndex + lastTerm = r.mu.lastTerm + raftLogSize = r.mu.raftLogSize + r.mu.RUnlock() + + // We refresh pending commands after applying a snapshot because this + // replica may have been temporarily partitioned from the Raft group and + // missed leadership changes that occurred. Suppose node A is the leader, + // and then node C gets partitioned away from the others. Leadership passes + // back and forth between A and B during the partition, but when the + // partition is healed node A is leader again. + if !r.store.TestingKnobs().DisableRefreshReasonSnapshotApplied && + refreshReason == noReason { + refreshReason = reasonSnapshotApplied + } } + } else if !raft.IsEmptySnap(rd.Snapshot) { + // If we didn't expect Raft to have a snapshot but it has one + // regardless, that is unexpected and indicates a programming + // error. + err := makeNonDeterministicFailure( + "have inSnap=nil, but raft has a snapshot %s", + raft.DescribeSnapshot(rd.Snapshot), + ) + return stats, getNonDeterministicFailureExplanation(err), err } // If the ready struct includes entries that have been committed, these @@ -1704,6 +1740,10 @@ func (r *Replica) acquireSplitLock( if err != nil { return nil, err } + // The right hand side of a split is always uninitialized since + // the left hand side blocks snapshots to it, and a snapshot is + // required to initialize it (if the split trigger doesn't - and + // this code here is part of the split trigger). if rightRepl.IsInitialized() { return nil, errors.Errorf("RHS of split %s / %s already initialized before split application", &split.LeftDesc, &split.RightDesc) diff --git a/pkg/kv/kvserver/replica_raftstorage.go b/pkg/kv/kvserver/replica_raftstorage.go index bd52b3dbdb09..bb7db644e325 100644 --- a/pkg/kv/kvserver/replica_raftstorage.go +++ b/pkg/kv/kvserver/replica_raftstorage.go @@ -15,7 +15,6 @@ import ( "fmt" "math" "strconv" - "sync/atomic" "time" "github.com/cockroachdb/cockroach/pkg/keys" @@ -515,6 +514,7 @@ type IncomingSnapshot struct { // See the comment on VersionUnreplicatedRaftTruncatedState for details. UsesUnreplicatedTruncatedState bool snapType SnapshotRequest_Type + placeholder *ReplicaPlaceholder } func (s *IncomingSnapshot) String() string { @@ -737,11 +737,11 @@ func clearRangeData( return nil } -// applySnapshot updates the replica and its store based on the given snapshot -// and associated HardState. All snapshots must pass through Raft for -// correctness, i.e. the parameters to this method must be taken from a -// raft.Ready. Any replicas specified in subsumedRepls will be destroyed -// atomically with the application of the snapshot. +// applySnapshot updates the replica and its store based on the given +// (non-empty) snapshot and associated HardState. All snapshots must pass +// through Raft for correctness, i.e. the parameters to this method must be +// taken from a raft.Ready. Any replicas specified in subsumedRepls will be +// destroyed atomically with the application of the snapshot. // // If there is a placeholder associated with r, applySnapshot will remove that // placeholder from the store if and only if it does not return an error. @@ -755,7 +755,7 @@ func clearRangeData( func (r *Replica) applySnapshot( ctx context.Context, inSnap IncomingSnapshot, - snap raftpb.Snapshot, + nonemptySnap raftpb.Snapshot, hs raftpb.HardState, subsumedRepls []*Replica, ) (err error) { @@ -792,25 +792,11 @@ func (r *Replica) applySnapshot( } }() - if raft.IsEmptySnap(snap) { - // Raft discarded the snapshot, indicating that our local state is - // already ahead of what the snapshot provides. But we count it for - // stats (see the defer above). - // - // Since we're not returning an error, we're responsible for removing any - // placeholder that might exist. - r.store.mu.Lock() - if r.store.removePlaceholderLocked(ctx, r.RangeID) { - atomic.AddInt32(&r.store.counts.filledPlaceholders, 1) - } - r.store.mu.Unlock() - return nil - } if raft.IsEmptyHardState(hs) { // Raft will never provide an empty HardState if it is providing a // nonempty snapshot because we discard snapshots that do not increase // the commit index. - log.Fatalf(ctx, "found empty HardState for non-empty Snapshot %+v", snap) + log.Fatalf(ctx, "found empty HardState for non-empty Snapshot %+v", nonemptySnap) } var stats struct { @@ -820,7 +806,7 @@ func (r *Replica) applySnapshot( ingestion time.Time } log.Infof(ctx, "applying snapshot of type %s [id=%s index=%d]", inSnap.snapType, - inSnap.SnapUUID.Short(), snap.Metadata.Index) + inSnap.SnapUUID.Short(), nonemptySnap.Metadata.Index) defer func(start time.Time) { now := timeutil.Now() totalLog := fmt.Sprintf( @@ -842,7 +828,7 @@ func (r *Replica) applySnapshot( ) log.Infof( ctx, "applied snapshot of type %s [%s%s%sid=%s index=%d]", inSnap.snapType, totalLog, - subsumedReplicasLog, ingestionLog, inSnap.SnapUUID.Short(), snap.Metadata.Index, + subsumedReplicasLog, ingestionLog, inSnap.SnapUUID.Short(), nonemptySnap.Metadata.Index, ) }(timeutil.Now()) @@ -909,9 +895,9 @@ func (r *Replica) applySnapshot( } } - if s.RaftAppliedIndex != snap.Metadata.Index { + if s.RaftAppliedIndex != nonemptySnap.Metadata.Index { log.Fatalf(ctx, "snapshot RaftAppliedIndex %d doesn't match its metadata index %d", - s.RaftAppliedIndex, snap.Metadata.Index) + s.RaftAppliedIndex, nonemptySnap.Metadata.Index) } if expLen := s.RaftAppliedIndex - s.TruncatedState.Index; expLen != uint64(len(inSnap.LogEntries)) { @@ -981,8 +967,11 @@ func (r *Replica) applySnapshot( r.store.mu.Lock() r.mu.Lock() - if r.store.removePlaceholderLocked(ctx, r.RangeID) { - atomic.AddInt32(&r.store.counts.filledPlaceholders, 1) + if inSnap.placeholder != nil { + _, err := r.store.removePlaceholderLocked(ctx, inSnap.placeholder, removePlaceholderFilled) + if err != nil { + log.Fatalf(ctx, "unable to remove placeholder: %s", err) + } } r.setDescLockedRaftMuLocked(ctx, s.Desc) if err := r.store.maybeMarkReplicaInitializedLockedReplLocked(ctx, r); err != nil { diff --git a/pkg/kv/kvserver/replicate_queue_test.go b/pkg/kv/kvserver/replicate_queue_test.go index 812b4d919fbc..4d5371cb2869 100644 --- a/pkg/kv/kvserver/replicate_queue_test.go +++ b/pkg/kv/kvserver/replicate_queue_test.go @@ -654,6 +654,7 @@ func getNonVoterNodeIDs(rangeDesc roachpb.RangeDescriptor) (result []roachpb.Nod // from voter to non-voter. func TestReplicateQueueSwapVotersWithNonVoters(t *testing.T) { defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) ctx := context.Background() serverArgs := make(map[int]base.TestServerArgs) diff --git a/pkg/kv/kvserver/store.go b/pkg/kv/kvserver/store.go index 0c6ff611b9cb..df25191631cf 100644 --- a/pkg/kv/kvserver/store.go +++ b/pkg/kv/kvserver/store.go @@ -604,12 +604,16 @@ type Store struct { } counts struct { - // Number of placeholders removed due to error. - removedPlaceholders int32 - // Number of placeholders successfully filled by a snapshot. + // Number of placeholders removed due to error. Not a good fit for meaningful + // metrics, as snapshots to initialized ranges don't get a placeholder. + failedPlaceholders int32 + // Number of placeholders successfully filled by a snapshot. Not a good fit + // for meaningful metrics, as snapshots to initialized ranges don't get a + // placeholder. filledPlaceholders int32 // Number of placeholders removed due to a snapshot that was dropped by - // raft. + // raft. Not a good fit for meaningful metrics, as snapshots to initialized + // ranges don't get a placeholder. droppedPlaceholders int32 } diff --git a/pkg/kv/kvserver/store_raft.go b/pkg/kv/kvserver/store_raft.go index 659521311465..c2a27d18aacc 100644 --- a/pkg/kv/kvserver/store_raft.go +++ b/pkg/kv/kvserver/store_raft.go @@ -12,7 +12,6 @@ package kvserver import ( "context" - "sync/atomic" "time" "unsafe" @@ -270,6 +269,9 @@ func (s *Store) processRaftRequestWithReplica( // handle any updated Raft Ready state. It also adds and later removes the // (potentially) necessary placeholder to protect against concurrent access to // the keyspace encompassed by the snapshot but not yet guarded by the replica. +// +// If (and only if) no error is returned, the placeholder (if any) in inSnap +// will have been removed. func (s *Store) processRaftSnapshotRequest( ctx context.Context, snapHeader *SnapshotRequest_Header, inSnap IncomingSnapshot, ) *roachpb.Error { @@ -285,52 +287,27 @@ func (s *Store) processRaftSnapshotRequest( log.Fatalf(ctx, "expected snapshot: %+v", snapHeader.RaftMessageRequest) } - // Check to see if a snapshot can be applied. Snapshots can always be applied - // to initialized replicas. Note that if we add a placeholder we need to - // already be holding Replica.raftMu in order to prevent concurrent - // raft-ready processing of uninitialized replicas. - var addedPlaceholder bool - var removePlaceholder bool - if err := func() error { - s.mu.Lock() - defer s.mu.Unlock() - placeholder, err := s.canApplySnapshotLocked(ctx, snapHeader) - if err != nil { - // If we cannot accept the snapshot, return an error before - // passing it to RawNode.Step, since our error handling options - // past that point are limited. - log.Infof(ctx, "cannot apply snapshot: %s", err) - return err - } - - if placeholder != nil { - // NB: The placeholder added here is either removed below after a - // preemptive snapshot is applied or after the next call to - // Replica.handleRaftReady. Note that we can only get here if the - // replica doesn't exist or is uninitialized. - if err := s.addPlaceholderLocked(placeholder); err != nil { - log.Fatalf(ctx, "could not add vetted placeholder %s: %+v", placeholder, err) + var stats handleRaftReadyStats + typ := removePlaceholderFailed + defer func() { + // In the typical case, handleRaftReadyRaftMuLocked calls through to + // applySnapshot which will apply the snapshot and also converts the + // placeholder entry (if any) to the now-initialized replica. However we + // may also error out below, or raft may also ignore the snapshot, and so + // the placeholder would remain. + // + // NB: it's unclear in which case we could actually get raft to ignore a + // snapshot attached to a placeholder. A placeholder existing implies that + // the snapshot is targeting an uninitialized replica. The only known reason + // for raft to ignore a snapshot is if it doesn't move the applied index + // forward, but an uninitialized replica's applied index is zero (and a + // snapshot's is at least raftInitialLogIndex). + if inSnap.placeholder != nil { + if _, err := s.removePlaceholder(ctx, inSnap.placeholder, typ); err != nil { + log.Fatalf(ctx, "unable to remove placeholder: %s", err) } - addedPlaceholder = true } - return nil - }(); err != nil { - return roachpb.NewError(err) - } - - if addedPlaceholder { - // If we added a placeholder remove it before we return unless some other - // part of the code takes ownership of the removal (indicated by setting - // removePlaceholder to false). - removePlaceholder = true - defer func() { - if removePlaceholder { - if s.removePlaceholder(ctx, snapHeader.RaftMessageRequest.RangeID) { - atomic.AddInt32(&s.counts.removedPlaceholders, 1) - } - } - }() - } + }() if snapHeader.RaftMessageRequest.Message.From == snapHeader.RaftMessageRequest.Message.To { // This is a special case exercised during recovery from loss of quorum. @@ -355,9 +332,24 @@ func (s *Store) processRaftSnapshotRequest( return roachpb.NewError(err) } - _, expl, err := r.handleRaftReadyRaftMuLocked(ctx, inSnap) + // We've handed the snapshot to Raft, which will typically apply it (in + // which case the placeholder, if any, is removed by the time + // handleRaftReadyRaftMuLocked returns. We handle the other case in a + // defer() above. Note that we could infer when the placeholder should still + // be there based on `stats.snap.applied` but it is a questionable use of + // stats and more susceptible to bugs. + typ = removePlaceholderDropped + var expl string + var err error + stats, expl, err = r.handleRaftReadyRaftMuLocked(ctx, inSnap) + if !stats.snap.applied { + // This line would be hit if a snapshot was sent when it isn't necessary + // (i.e. follower was able to catch up via the log in the interim) or when + // multiple snapshots raced (as is possible when raft leadership changes + // and both the old and new leaders send snapshots). + log.Infof(ctx, "ignored stale snapshot at index %d", snapHeader.RaftMessageRequest.Message.Snapshot.Metadata.Index) + } maybeFatalOnRaftReadyErr(ctx, expl, err) - removePlaceholder = false return nil }) } @@ -521,7 +513,7 @@ func (s *Store) processReady(ctx context.Context, rangeID roachpb.RangeID) { ctx = r.raftSchedulerCtx(ctx) start := timeutil.Now() stats, expl, err := r.handleRaftReady(ctx, noSnap) - removed := maybeFatalOnRaftReadyErr(ctx, expl, err) + maybeFatalOnRaftReadyErr(ctx, expl, err) elapsed := timeutil.Since(start) s.metrics.RaftWorkingDurationNanos.Inc(elapsed.Nanoseconds()) s.metrics.RaftHandleReadyLatency.RecordValue(elapsed.Nanoseconds()) @@ -533,21 +525,6 @@ func (s *Store) processReady(ctx context.Context, rangeID roachpb.RangeID) { log.Warningf(ctx, "handle raft ready: %.1fs [applied=%d, batches=%d, state_assertions=%d]", elapsed.Seconds(), stats.entriesProcessed, stats.batchesProcessed, stats.stateAssertions) } - if !removed && !r.IsInitialized() { - // Only an uninitialized replica can have a placeholder since, by - // definition, an initialized replica will be present in the - // replicasByKey map. While the replica will usually consume the - // placeholder itself, that isn't guaranteed and so this invocation - // here is crucial (i.e. don't remove it). - // - // We need to hold raftMu here to prevent removing a placeholder that is - // actively being used by Store.processRaftRequest. - r.raftMu.Lock() - if s.removePlaceholder(ctx, r.RangeID) { - atomic.AddInt32(&s.counts.droppedPlaceholders, 1) - } - r.raftMu.Unlock() - } } func (s *Store) processTick(ctx context.Context, rangeID roachpb.RangeID) bool { diff --git a/pkg/kv/kvserver/store_remove_replica.go b/pkg/kv/kvserver/store_remove_replica.go index 43e28ac9b71c..6350c6364755 100644 --- a/pkg/kv/kvserver/store_remove_replica.go +++ b/pkg/kv/kvserver/store_remove_replica.go @@ -167,6 +167,12 @@ func (s *Store) removeInitializedReplicaRaftMuLocked( s.mu.Lock() defer s.mu.Unlock() s.unlinkReplicaByRangeIDLocked(ctx, rep.RangeID) + // There can't be a placeholder, as the replica is still in replicasByKey + // and it is initialized. (A placeholder would also be in replicasByKey + // and overlap the replica, which is impossible). + if ph, ok := s.mu.replicaPlaceholders[rep.RangeID]; ok { + log.Fatalf(ctx, "initialized replica %s unexpectedly had a placeholder: %+v", rep, ph) + } if it := s.mu.replicasByKey.DeleteReplica(ctx, rep); it.repl != rep { // We already checked that our replica was present in replicasByKey // above. Nothing should have been able to change that. @@ -175,7 +181,6 @@ func (s *Store) removeInitializedReplicaRaftMuLocked( if it := s.getOverlappingKeyRangeLocked(desc); it.item != nil { log.Fatalf(ctx, "corrupted replicasByKey map: %s and %s overlapped", rep, it.item) } - delete(s.mu.replicaPlaceholders, rep.RangeID) // TODO(peter): Could release s.mu.Lock() here. s.maybeGossipOnCapacityChange(ctx, rangeRemoveEvent) s.scanner.RemoveReplica(rep) @@ -241,14 +246,6 @@ func (s *Store) removeUninitializedReplicaRaftMuLocked( log.Fatalf(ctx, "uninitialized replica %v was unexpectedly replaced", existing) } - // Only an uninitialized replica can have a placeholder since, by - // definition, an initialized replica will be present in the - // replicasByKey map. While the replica will usually consume the - // placeholder itself, that isn't guaranteed and so this invocation - // here is crucial (i.e. don't remove it). - if s.removePlaceholderLocked(ctx, rep.RangeID) { - atomic.AddInt32(&s.counts.droppedPlaceholders, 1) - } s.unlinkReplicaByRangeIDLocked(ctx, rep.RangeID) } @@ -268,30 +265,113 @@ func (s *Store) unlinkReplicaByRangeIDLocked(ctx context.Context, rangeID roachp s.unregisterLeaseholderByID(ctx, rangeID) } -// removePlaceholder removes a placeholder for the specified range if it -// exists, returning true if a placeholder was present and removed and false -// otherwise. Requires that the raftMu of the replica whose place is being held -// is locked. -func (s *Store) removePlaceholder(ctx context.Context, rngID roachpb.RangeID) bool { +// removePlaceholder removes a placeholder for the specified range. +// Requires that the raftMu of the replica whose place is being held +// is locked. See removePlaceholderType for existence semantics. +func (s *Store) removePlaceholder( + ctx context.Context, ph *ReplicaPlaceholder, typ removePlaceholderType, +) (removed bool, _ error) { s.mu.Lock() defer s.mu.Unlock() - return s.removePlaceholderLocked(ctx, rngID) + return s.removePlaceholderLocked(ctx, ph, typ) } -// removePlaceholderLocked removes the specified placeholder. Requires that -// Store.mu and the raftMu of the replica whose place is being held are locked. -func (s *Store) removePlaceholderLocked(ctx context.Context, rngID roachpb.RangeID) bool { +type removePlaceholderType byte + +const ( + // The placeholder was filled, i.e. the snapshot was applied successfully. + // This is only legal to use when the placeholder exists, and so in particular + // it can't be invoked multiple times. + removePlaceholderFilled removePlaceholderType = iota + // Raft didn't apply snapshot. Note that this is only counting snapshots + // that raft dropped in the presence of a placeholder, which means snapshots + // that targeted an uninitialized replica. There is currently no reason for + // those to ever be dropped, if anything snapshots that get dropped would + // target an already initialized replica, but those will not augment the + // metric related to this const. + // + // This type allows idempotent deletion, i.e. it can be invoked even if the + // placeholder has already been filled or removed before, simplifying the + // cleanup on failed operations. + removePlaceholderDropped + // The snapshot never got to raft, i.e. failed during receipt, for example. + // This does not account for failed snapshots targeting initialized replicas. + // + // This type allows idempotent deletion, i.e. it can be invoked even if the + // placeholder has already been filled or removed before, simplifying the + // cleanup on failed operations. + removePlaceholderFailed +) + +// removePlaceholderLocked removes a placeholder for the specified range. +// Requires that the raftMu of the replica whose place is being held +// is locked. See removePlaceholderType for existence semantics. +// +// If typ is removePlaceholderFilled, an error is returned unless `removed` +// is true. For the other types, removal is idempotent. +func (s *Store) removePlaceholderLocked( + ctx context.Context, inPH *ReplicaPlaceholder, typ removePlaceholderType, +) (removed bool, _ error) { + wasTainted := !atomic.CompareAndSwapInt32(&inPH.tainted, 0, 1) + idempotent := wasTainted && typ == removePlaceholderFailed || typ == removePlaceholderDropped + if wasTainted { + // The placeholder was already tainted, i.e. it was passed to + // removePlaceholderLocked before. + if !idempotent { + // We need the placeholder to exist, but it's already gone. This is a bug. + return false, errors.AssertionFailedf( + "attempt to remove already removed placeholder %+v", inPH, + ) + } + // Continue so that we can verify that the placeholder is indeed gone. + } + + rngID := inPH.Desc().RangeID placeholder, ok := s.mu.replicaPlaceholders[rngID] + + if wasTainted != !ok { + return false, errors.AssertionFailedf("expected placeholder to exist: %t but found in store: %t", !wasTainted, ok) + } + if !ok { - return false + // The placeholder is not in the store, but that's ok since wasTainted is + // thus true (since we got here) and this implies that idempotent==true + // (again since we made it here). + return false, nil + } + + if placeholder != inPH { + if idempotent { + // In idempotent mode, the placeholder was not in the store before, so + // nothing would have prevented another placeholder for the same range to + // slip in. + return false, nil + } + // The placeholder acts as a lock, and when we're filling or dropping it we + // only do so once, so how would we now see a different placeholder from the + // one we previously inserted? There must be a bug. + return false, errors.AssertionFailedf( + "placeholder %s is being dropped or filled, but store has conflicting placeholder %s", + inPH, placeholder, + ) } + + // Remove the placeholder from the store. + if it := s.mu.replicasByKey.DeletePlaceholder(ctx, placeholder); it.ph != placeholder { - log.Fatalf(ctx, "r%d: placeholder %v not found, got %+v", rngID, placeholder, it) - return true // unreachable + return false, errors.AssertionFailedf("placeholder %v not found, got %+v", placeholder, it) } delete(s.mu.replicaPlaceholders, rngID) if it := s.getOverlappingKeyRangeLocked(&placeholder.rangeDesc); it.item != nil { - log.Fatalf(ctx, "corrupted replicasByKey map: %s and %s overlapped", it.ph, it.item) + return false, errors.AssertionFailedf("corrupted replicasByKey map: %s and %s overlapped", it.ph, it.item) + } + switch typ { + case removePlaceholderDropped: + atomic.AddInt32(&s.counts.droppedPlaceholders, 1) + case removePlaceholderFailed: + atomic.AddInt32(&s.counts.failedPlaceholders, 1) + case removePlaceholderFilled: + atomic.AddInt32(&s.counts.filledPlaceholders, 1) } - return true + return true, nil } diff --git a/pkg/kv/kvserver/store_snapshot.go b/pkg/kv/kvserver/store_snapshot.go index 90a5e751a597..ca47ab55677e 100644 --- a/pkg/kv/kvserver/store_snapshot.go +++ b/pkg/kv/kvserver/store_snapshot.go @@ -46,7 +46,7 @@ const ( storeDrainingMsg = "store is draining" // IntersectingSnapshotMsg is part of the error message returned from - // canApplySnapshotLocked and is exposed here so testing can rely on it. + // canAcceptSnapshotLocked and is exposed here so testing can rely on it. IntersectingSnapshotMsg = "snapshot intersects existing range" ) @@ -581,14 +581,14 @@ func (s *Store) reserveSnapshot( }, "", nil } -// canApplySnapshotLocked returns (_, nil) if the snapshot can be applied to +// canAcceptSnapshotLocked returns (_, nil) if the snapshot can be applied to // this store's replica (i.e. the snapshot is not from an older incarnation of -// the replica) and a placeholder can be added to the replicasByKey map (if -// necessary). If a placeholder is required, it is returned as the first value. +// the replica) and a placeholder that can be (but is not yet) added to the +// replicasByKey map (if necessary). // -// Both the store mu (and the raft mu for an existing replica if there is one) +// Both the store mu and the raft mu for the existing replica (which must exist) // must be held. -func (s *Store) canApplySnapshotLocked( +func (s *Store) canAcceptSnapshotLocked( ctx context.Context, snapHeader *SnapshotRequest_Header, ) (*ReplicaPlaceholder, error) { if snapHeader.IsPreemptive() { @@ -604,7 +604,7 @@ func (s *Store) canApplySnapshotLocked( int64(desc.RangeID), ) if !ok { - return nil, errors.Errorf("canApplySnapshotLocked requires a replica present") + return nil, errors.Errorf("canAcceptSnapshotLocked requires a replica present") } existingRepl := (*Replica)(v) // The raftMu is held which allows us to use the existing replica as a @@ -662,7 +662,7 @@ func (s *Store) checkSnapshotOverlapLocked( // NB: this check seems redundant since placeholders are also represented in // replicasByKey (and thus returned in getOverlappingKeyRangeLocked). if exRng, ok := s.mu.replicaPlaceholders[desc.RangeID]; ok { - return errors.Errorf("%s: canApplySnapshotLocked: cannot add placeholder, have an existing placeholder %s %v", s, exRng, snapHeader.RaftMessageRequest.FromReplica) + return errors.Errorf("%s: canAcceptSnapshotLocked: cannot add placeholder, have an existing placeholder %s %v", s, exRng, snapHeader.RaftMessageRequest.FromReplica) } // TODO(benesch): consider discovering and GC'ing *all* overlapping ranges, @@ -712,37 +712,6 @@ func (s *Store) checkSnapshotOverlapLocked( return nil } -// shouldAcceptSnapshotData is an optimization to check whether we should even -// bother to read the data for an incoming snapshot. If the snapshot overlaps an -// existing replica or placeholder, we'd error during application anyway, so do -// it before transferring all the data. This method is a guess and may have -// false positives. If the snapshot should be rejected, an error is returned -// with a description of why. Otherwise, nil means we should accept the -// snapshot. -func (s *Store) shouldAcceptSnapshotData( - ctx context.Context, snapHeader *SnapshotRequest_Header, -) error { - if snapHeader.IsPreemptive() { - return errors.AssertionFailedf(`expected a raft or learner snapshot`) - } - pErr := s.withReplicaForRequest( - ctx, &snapHeader.RaftMessageRequest, func(ctx context.Context, r *Replica) *roachpb.Error { - ctx = r.AnnotateCtx(ctx) - // If the current replica is not initialized then we should accept this - // snapshot if it doesn't overlap existing ranges. - if !r.IsInitialized() { - s.mu.Lock() - defer s.mu.Unlock() - return roachpb.NewError(s.checkSnapshotOverlapLocked(ctx, snapHeader)) - } - // If the current range is initialized then we need to accept this - // snapshot. - return nil - }, - ) - return pErr.GoError() -} - // receiveSnapshot receives an incoming snapshot via a pre-opened GRPC stream. func (s *Store) receiveSnapshot( ctx context.Context, header *SnapshotRequest_Header, stream incomingSnapshotStream, @@ -777,17 +746,41 @@ func (s *Store) receiveSnapshot( } defer cleanup() - // Check to see if the snapshot can be applied but don't attempt to add - // a placeholder here, because we're not holding the replica's raftMu. - // We'll perform this check again later after receiving the rest of the - // snapshot data - this is purely an optimization to prevent downloading - // a snapshot that we know we won't be able to apply. - if err := s.shouldAcceptSnapshotData(ctx, header); err != nil { - return sendSnapshotError(stream, - errors.Wrapf(err, "%s,r%d: cannot apply snapshot", s, header.State.Desc.RangeID), - ) + // The comment on ReplicaPlaceholder motivates and documents + // ReplicaPlaceholder semantics. Please be familiar with them + // before making any changes. + var placeholder *ReplicaPlaceholder + if pErr := s.withReplicaForRequest( + ctx, &header.RaftMessageRequest, func(ctx context.Context, r *Replica, + ) *roachpb.Error { + var err error + s.mu.Lock() + defer s.mu.Unlock() + placeholder, err = s.canAcceptSnapshotLocked(ctx, header) + if err != nil { + return roachpb.NewError(err) + } + if placeholder != nil { + if err := s.addPlaceholderLocked(placeholder); err != nil { + return roachpb.NewError(err) + } + } + return nil + }); pErr != nil { + log.Infof(ctx, "cannot accept snapshot: %s", pErr) + return pErr.GoError() } + defer func() { + if placeholder != nil { + // Remove the placeholder, if it's still there. Most of the time it will + // have been filled and this is a no-op. + if _, err := s.removePlaceholder(ctx, placeholder, removePlaceholderFailed); err != nil { + log.Fatalf(ctx, "unable to remove placeholder: %s", err) + } + } + }() + // Determine which snapshot strategy the sender is using to send this // snapshot. If we don't know how to handle the specified strategy, return // an error. @@ -823,10 +816,10 @@ func (s *Store) receiveSnapshot( if err != nil { return err } + inSnap.placeholder = placeholder if err := s.processRaftSnapshotRequest(ctx, header, inSnap); err != nil { return sendSnapshotError(stream, errors.Wrap(err.GoError(), "failed to apply snapshot")) } - return stream.Send(&SnapshotResponse{Status: SnapshotResponse_APPLIED}) } diff --git a/pkg/kv/kvserver/store_test.go b/pkg/kv/kvserver/store_test.go index ad4a0520f079..d54369e574f1 100644 --- a/pkg/kv/kvserver/store_test.go +++ b/pkg/kv/kvserver/store_test.go @@ -18,6 +18,7 @@ import ( "math/rand" "reflect" "sort" + "strings" "sync" "sync/atomic" "testing" @@ -2636,37 +2637,72 @@ func TestStoreRangePlaceholders(t *testing.T) { }, } + check := func(exp string) { + exp = strings.TrimSpace(exp) + t.Helper() + act := strings.TrimSpace(s.mu.replicasByKey.String()) + require.Equal(t, exp, act) + } + s.mu.Lock() defer s.mu.Unlock() // Test that simple insertion works. - if err := s.addPlaceholderLocked(placeholder1); err != nil { - t.Fatalf("could not add placeholder to empty store, got %s", err) + require.NoError(t, s.addPlaceholderLocked(placeholder1)) + require.NoError(t, s.addPlaceholderLocked(placeholder2)) + + check(` +[] - [99] (/Min - "c") +[99] - [100] ("c" - "d") +[100] - [255 255] ("d" - /Max)`) + + checkErr := func(re string) func(bool, error) { + t.Helper() + return func(removed bool, err error) { + require.True(t, testutils.IsError(err, re), "expected to match %s: %v", re, err) + require.False(t, removed) + } + } + checkRemoved := func(removed bool, nilErr error) { + t.Helper() + require.NoError(t, nilErr) + require.True(t, removed) } - if err := s.addPlaceholderLocked(placeholder2); err != nil { - t.Fatalf("could not add non-overlapping placeholder, got %s", err) + checkNotRemoved := func(removed bool, nilErr error) { + t.Helper() + require.NoError(t, nilErr) + require.False(t, removed) } // Test that simple deletion works. - if !s.removePlaceholderLocked(ctx, placeholder1.rangeDesc.RangeID) { - t.Fatalf("could not remove placeholder that was present") - } + checkRemoved(s.removePlaceholderLocked(ctx, placeholder1, removePlaceholderFailed)) + + check(` +[] - [99] (/Min - "c") +[100] - [255 255] ("d" - /Max)`) // Test cannot double insert the same placeholder. - if err := s.addPlaceholderLocked(placeholder1); err != nil { - t.Fatalf("could not re-add placeholder after removal, got %s", err) - } + placeholder1.tainted = 0 // reset for re-use + require.NoError(t, s.addPlaceholderLocked(placeholder1)) if err := s.addPlaceholderLocked(placeholder1); !testutils.IsError(err, ".*overlaps with existing") { t.Fatalf("should not be able to add ReplicaPlaceholder for the same key twice, got: %+v", err) } - // Test cannot double delete a placeholder. - if !s.removePlaceholderLocked(ctx, placeholder1.rangeDesc.RangeID) { - t.Fatalf("could not remove placeholder that was present") - } - if s.removePlaceholderLocked(ctx, placeholder1.rangeDesc.RangeID) { - t.Fatalf("successfully removed placeholder that was not present") - } + // Test double deletion of a placeholder. + checkRemoved(s.removePlaceholderLocked(ctx, placeholder1, removePlaceholderFilled)) + checkNotRemoved(s.removePlaceholderLocked(ctx, placeholder1, removePlaceholderFailed)) + checkNotRemoved(s.removePlaceholderLocked(ctx, placeholder1, removePlaceholderDropped)) + checkErr(`attempt to remove already removed placeholder`)(s.removePlaceholderLocked( + ctx, placeholder1, removePlaceholderFilled, + )) + placeholder1.tainted = 0 // pretend it wasn't already deleted + checkErr(`expected placeholder to exist: true but found in store: false`)(s.removePlaceholderLocked( + ctx, placeholder1, removePlaceholderDropped, + )) + + check(` +[] - [99] (/Min - "c") +[100] - [255 255] ("d" - /Max)`) // This placeholder overlaps with an existing replica. placeholder1 = &ReplicaPlaceholder{ @@ -2677,15 +2713,20 @@ func TestStoreRangePlaceholders(t *testing.T) { }, } - // Test that placeholder cannot clobber existing replica. - if err := s.addPlaceholderLocked(placeholder1); !testutils.IsError(err, ".*overlaps with existing") { - t.Fatalf("should not be able to add ReplicaPlaceholder when Replica already exists, got: %+v", err) + addPH := func(ph *ReplicaPlaceholder) (bool, error) { + return false, s.addPlaceholderLocked(ph) } + // Test that placeholder cannot clobber existing replica. + checkErr(`.*overlaps with existing`)(addPH(placeholder1)) + // Test that Placeholder deletion doesn't delete replicas. - if s.removePlaceholderLocked(ctx, repID) { - t.Fatalf("should not be able to process removeReplicaPlaceholder for a RangeID where a Replica exists") - } + placeholder1.tainted = 0 + checkErr(`expected placeholder to exist: true`)(s.removePlaceholderLocked(ctx, placeholder1, removePlaceholderFilled)) + checkNotRemoved(s.removePlaceholderLocked(ctx, placeholder1, removePlaceholderFailed)) + check(` +[] - [99] (/Min - "c") +[100] - [255 255] ("d" - /Max)`) } // Test that we remove snapshot placeholders when raft ignores the @@ -2763,11 +2804,23 @@ func TestStoreRemovePlaceholderOnRaftIgnored(t *testing.T) { }, }, } + + placeholder := &ReplicaPlaceholder{rangeDesc: *repl1.Desc()} + + { + s.mu.Lock() + err := s.addPlaceholderLocked(placeholder) + s.mu.Unlock() + require.NoError(t, err) + } + if err := s.processRaftSnapshotRequest(ctx, req, IncomingSnapshot{ - SnapUUID: uuid.MakeV4(), - State: &kvserverpb.ReplicaState{Desc: repl1.Desc()}, - }); err != nil { + SnapUUID: uuid.MakeV4(), + State: &kvserverpb.ReplicaState{Desc: repl1.Desc()}, + placeholder: placeholder, + }, + ); err != nil { t.Fatal(err) } diff --git a/pkg/testutils/testcluster/testcluster.go b/pkg/testutils/testcluster/testcluster.go index 54e90e19b74f..b4432a038bd2 100644 --- a/pkg/testutils/testcluster/testcluster.go +++ b/pkg/testutils/testcluster/testcluster.go @@ -592,18 +592,41 @@ func (tc *TestCluster) Targets(serverIdxs ...int) []roachpb.ReplicationTarget { func (tc *TestCluster) changeReplicas( changeType roachpb.ReplicaChangeType, startKey roachpb.RKey, targets ...roachpb.ReplicationTarget, ) (roachpb.RangeDescriptor, error) { + tc.t.Helper() ctx := context.TODO() - var beforeDesc roachpb.RangeDescriptor - if err := tc.Servers[0].DB().GetProto( - ctx, keys.RangeDescriptorKey(startKey), &beforeDesc, - ); err != nil { - return roachpb.RangeDescriptor{}, errors.Wrap(err, "range descriptor lookup error") + + var returnErr error + var desc *roachpb.RangeDescriptor + if err := testutils.SucceedsSoonError(func() error { + tc.t.Helper() + var beforeDesc roachpb.RangeDescriptor + if err := tc.Servers[0].DB().GetProto( + ctx, keys.RangeDescriptorKey(startKey), &beforeDesc, + ); err != nil { + return errors.Wrap(err, "range descriptor lookup error") + } + var err error + desc, err = tc.Servers[0].DB().AdminChangeReplicas( + ctx, startKey.AsRawKey(), beforeDesc, roachpb.MakeReplicationChanges(changeType, targets...), + ) + if kvserver.IsRetriableReplicationChangeError(err) { + tc.t.Logf("encountered retriable replication change error: %v", err) + return err + } + // Don't return blindly - if this isn't an error we think is related to a + // replication error that we can retry, save the error to the outer scope + // and return nil. + returnErr = err + return nil + }); err != nil { + returnErr = err } - desc, err := tc.Servers[0].DB().AdminChangeReplicas( - ctx, startKey.AsRawKey(), beforeDesc, roachpb.MakeReplicationChanges(changeType, targets...), - ) - if err != nil { - return roachpb.RangeDescriptor{}, errors.Wrap(err, "AdminChangeReplicas error") + + if returnErr != nil { + // We mark the error as Handled so that tests that wanted the error in the + // first attempt but spent a while spinning in the retry loop above will + // fail. These should invoke ChangeReplicas directly. + return roachpb.RangeDescriptor{}, errors.Handled(errors.Wrap(returnErr, "AdminChangeReplicas error")) } return *desc, nil }