From e1f30c66effd61371c37a01d4b5ce025aef77142 Mon Sep 17 00:00:00 2001 From: Raphael 'kena' Poss Date: Thu, 4 Nov 2021 16:57:08 +0100 Subject: [PATCH] server,storage: capture filesystem properties in store descriptors We have found numerous times during troubleshooting that it is useful to inspect the filesystem type and mount options and, when hosted without containers, the block device used to support the filesystem. This patch achieves this by retrieving the filesystem details and attaching them to store descriptors. Release note: None --- pkg/kv/kvserver/store.go | 16 +- pkg/roachpb/metadata.go | 14 + pkg/roachpb/metadata.pb.go | 817 ++++++++++++++++++++++++++++---- pkg/roachpb/metadata.proto | 29 ++ pkg/server/config.go | 2 + pkg/storage/BUILD.bazel | 2 + pkg/storage/engine.go | 2 + pkg/storage/pebble.go | 11 +- pkg/storage/store_properties.go | 99 ++++ 9 files changed, 883 insertions(+), 109 deletions(-) create mode 100644 pkg/storage/store_properties.go diff --git a/pkg/kv/kvserver/store.go b/pkg/kv/kvserver/store.go index 4ef95d32a143..5468048a7a35 100644 --- a/pkg/kv/kvserver/store.go +++ b/pkg/kv/kvserver/store.go @@ -79,7 +79,7 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/logtags" "github.com/cockroachdb/redact" - "go.etcd.io/etcd/raft/v3" + raft "go.etcd.io/etcd/raft/v3" "golang.org/x/time/rate" ) @@ -2510,6 +2510,11 @@ func (s *Store) Attrs() roachpb.Attributes { return s.engine.Attrs() } +// Properties returns the properties of the underlying store. +func (s *Store) Properties() roachpb.StoreProperties { + return s.engine.Properties() +} + // Capacity returns the capacity of the underlying storage engine. Note that // this does not include reservations. // Note that Capacity() has the side effect of updating some of the store's @@ -2615,10 +2620,11 @@ func (s *Store) Descriptor(ctx context.Context, useCached bool) (*roachpb.StoreD // Initialize the store descriptor. return &roachpb.StoreDescriptor{ - StoreID: s.Ident.StoreID, - Attrs: s.Attrs(), - Node: *s.nodeDesc, - Capacity: capacity, + StoreID: s.Ident.StoreID, + Attrs: s.Attrs(), + Node: *s.nodeDesc, + Capacity: capacity, + Properties: s.Properties(), }, nil } diff --git a/pkg/roachpb/metadata.go b/pkg/roachpb/metadata.go index 4bff2db29088..add285b824cc 100644 --- a/pkg/roachpb/metadata.go +++ b/pkg/roachpb/metadata.go @@ -526,6 +526,20 @@ func (p Percentiles) SafeFormat(w redact.SafePrinter, _ rune) { p.P10, p.P25, p.P50, p.P75, p.P90, p.PMax) } +func (sc FileStoreProperties) String() string { + return redact.StringWithoutMarkers(sc) +} + +// SafeFormat implements the redact.SafeFormatter interface. +func (sc FileStoreProperties) SafeFormat(w redact.SafePrinter, _ rune) { + w.Printf("{path=%s, fs=%s, blkdev=%s, mnt=%s opts=%s}", + sc.Path, + redact.SafeString(sc.FsType), + sc.BlockDevice, + sc.MountPoint, + sc.MountOptions) +} + // String returns a string representation of the StoreCapacity. func (sc StoreCapacity) String() string { return redact.StringWithoutMarkers(sc) diff --git a/pkg/roachpb/metadata.pb.go b/pkg/roachpb/metadata.pb.go index 75525a52231c..1858b5e6b538 100644 --- a/pkg/roachpb/metadata.pb.go +++ b/pkg/roachpb/metadata.pb.go @@ -548,6 +548,88 @@ func (m *StoreCapacity) XXX_DiscardUnknown() { var xxx_messageInfo_StoreCapacity proto.InternalMessageInfo +// StoreProperties contains configuration and OS-level details for a storage device. +type StoreProperties struct { + // encrypted indicates whether the store is encrypted. + Encrypted bool `protobuf:"varint,1,opt,name=encrypted" json:"encrypted"` + // read_only indicates whether the store is attached read_only. + ReadOnly bool `protobuf:"varint,2,opt,name=read_only,json=readOnly" json:"read_only"` + // disk_properties reports details about the underlying filesystem, + // when the store is supported by a file store. Unset otherwise. + FileStoreProperties *FileStoreProperties `protobuf:"bytes,3,opt,name=file_store_properties,json=fileStoreProperties" json:"file_store_properties,omitempty"` +} + +func (m *StoreProperties) Reset() { *m = StoreProperties{} } +func (m *StoreProperties) String() string { return proto.CompactTextString(m) } +func (*StoreProperties) ProtoMessage() {} +func (*StoreProperties) Descriptor() ([]byte, []int) { + return fileDescriptor_ecd7f5cc803f82c8, []int{7} +} +func (m *StoreProperties) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *StoreProperties) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *StoreProperties) XXX_Merge(src proto.Message) { + xxx_messageInfo_StoreProperties.Merge(m, src) +} +func (m *StoreProperties) XXX_Size() int { + return m.Size() +} +func (m *StoreProperties) XXX_DiscardUnknown() { + xxx_messageInfo_StoreProperties.DiscardUnknown(m) +} + +var xxx_messageInfo_StoreProperties proto.InternalMessageInfo + +// FileStoreProperties contains configuration and OS-level details for a file store. +type FileStoreProperties struct { + // path reports the configured filesystem path for the store. + Path string `protobuf:"bytes,1,opt,name=path" json:"path"` + // fs_type reports the external filesystem type (ufs, ext4, etc), if known. + FsType string `protobuf:"bytes,2,opt,name=fs_type,json=fsType" json:"fs_type"` + // block_device reports which block devices supports the filesystem, if known. + BlockDevice string `protobuf:"bytes,3,opt,name=block_device,json=blockDevice" json:"block_device"` + // mount_point reports the mount point of the filesystem, if known. + MountPoint string `protobuf:"bytes,4,opt,name=mount_point,json=mountPoint" json:"mount_point"` + // mount_options reports the mount options, if known. + MountOptions string `protobuf:"bytes,5,opt,name=mount_options,json=mountOptions" json:"mount_options"` +} + +func (m *FileStoreProperties) Reset() { *m = FileStoreProperties{} } +func (*FileStoreProperties) ProtoMessage() {} +func (*FileStoreProperties) Descriptor() ([]byte, []int) { + return fileDescriptor_ecd7f5cc803f82c8, []int{8} +} +func (m *FileStoreProperties) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *FileStoreProperties) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil +} +func (m *FileStoreProperties) XXX_Merge(src proto.Message) { + xxx_messageInfo_FileStoreProperties.Merge(m, src) +} +func (m *FileStoreProperties) XXX_Size() int { + return m.Size() +} +func (m *FileStoreProperties) XXX_DiscardUnknown() { + xxx_messageInfo_FileStoreProperties.DiscardUnknown(m) +} + +var xxx_messageInfo_FileStoreProperties proto.InternalMessageInfo + // NodeDescriptor holds details on node physical/network topology. type NodeDescriptor struct { NodeID NodeID `protobuf:"varint,1,opt,name=node_id,json=nodeId,casttype=NodeID" json:"node_id"` @@ -568,7 +650,7 @@ func (m *NodeDescriptor) Reset() { *m = NodeDescriptor{} } func (m *NodeDescriptor) String() string { return proto.CompactTextString(m) } func (*NodeDescriptor) ProtoMessage() {} func (*NodeDescriptor) Descriptor() ([]byte, []int) { - return fileDescriptor_ecd7f5cc803f82c8, []int{7} + return fileDescriptor_ecd7f5cc803f82c8, []int{9} } func (m *NodeDescriptor) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -604,7 +686,7 @@ func (m *LocalityAddress) Reset() { *m = LocalityAddress{} } func (m *LocalityAddress) String() string { return proto.CompactTextString(m) } func (*LocalityAddress) ProtoMessage() {} func (*LocalityAddress) Descriptor() ([]byte, []int) { - return fileDescriptor_ecd7f5cc803f82c8, []int{8} + return fileDescriptor_ecd7f5cc803f82c8, []int{10} } func (m *LocalityAddress) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -632,17 +714,18 @@ var xxx_messageInfo_LocalityAddress proto.InternalMessageInfo // StoreDescriptor holds store information including store attributes, node // descriptor and store capacity. type StoreDescriptor struct { - StoreID StoreID `protobuf:"varint,1,opt,name=store_id,json=storeId,casttype=StoreID" json:"store_id"` - Attrs Attributes `protobuf:"bytes,2,opt,name=attrs" json:"attrs"` - Node NodeDescriptor `protobuf:"bytes,3,opt,name=node" json:"node"` - Capacity StoreCapacity `protobuf:"bytes,4,opt,name=capacity" json:"capacity"` + StoreID StoreID `protobuf:"varint,1,opt,name=store_id,json=storeId,casttype=StoreID" json:"store_id"` + Attrs Attributes `protobuf:"bytes,2,opt,name=attrs" json:"attrs"` + Node NodeDescriptor `protobuf:"bytes,3,opt,name=node" json:"node"` + Capacity StoreCapacity `protobuf:"bytes,4,opt,name=capacity" json:"capacity"` + Properties StoreProperties `protobuf:"bytes,5,opt,name=properties" json:"properties"` } func (m *StoreDescriptor) Reset() { *m = StoreDescriptor{} } func (m *StoreDescriptor) String() string { return proto.CompactTextString(m) } func (*StoreDescriptor) ProtoMessage() {} func (*StoreDescriptor) Descriptor() ([]byte, []int) { - return fileDescriptor_ecd7f5cc803f82c8, []int{9} + return fileDescriptor_ecd7f5cc803f82c8, []int{11} } func (m *StoreDescriptor) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -679,7 +762,7 @@ func (m *StoreDeadReplicas) Reset() { *m = StoreDeadReplicas{} } func (m *StoreDeadReplicas) String() string { return proto.CompactTextString(m) } func (*StoreDeadReplicas) ProtoMessage() {} func (*StoreDeadReplicas) Descriptor() ([]byte, []int) { - return fileDescriptor_ecd7f5cc803f82c8, []int{10} + return fileDescriptor_ecd7f5cc803f82c8, []int{12} } func (m *StoreDeadReplicas) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -713,7 +796,7 @@ type Locality struct { func (m *Locality) Reset() { *m = Locality{} } func (*Locality) ProtoMessage() {} func (*Locality) Descriptor() ([]byte, []int) { - return fileDescriptor_ecd7f5cc803f82c8, []int{11} + return fileDescriptor_ecd7f5cc803f82c8, []int{13} } func (m *Locality) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -749,7 +832,7 @@ type Tier struct { func (m *Tier) Reset() { *m = Tier{} } func (*Tier) ProtoMessage() {} func (*Tier) Descriptor() ([]byte, []int) { - return fileDescriptor_ecd7f5cc803f82c8, []int{12} + return fileDescriptor_ecd7f5cc803f82c8, []int{14} } func (m *Tier) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -789,7 +872,7 @@ type Version struct { func (m *Version) Reset() { *m = Version{} } func (*Version) ProtoMessage() {} func (*Version) Descriptor() ([]byte, []int) { - return fileDescriptor_ecd7f5cc803f82c8, []int{13} + return fileDescriptor_ecd7f5cc803f82c8, []int{15} } func (m *Version) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -823,6 +906,8 @@ func init() { proto.RegisterType((*RangeDescriptor)(nil), "cockroach.roachpb.RangeDescriptor") proto.RegisterType((*Percentiles)(nil), "cockroach.roachpb.Percentiles") proto.RegisterType((*StoreCapacity)(nil), "cockroach.roachpb.StoreCapacity") + proto.RegisterType((*StoreProperties)(nil), "cockroach.roachpb.StoreProperties") + proto.RegisterType((*FileStoreProperties)(nil), "cockroach.roachpb.FileStoreProperties") proto.RegisterType((*NodeDescriptor)(nil), "cockroach.roachpb.NodeDescriptor") proto.RegisterType((*LocalityAddress)(nil), "cockroach.roachpb.LocalityAddress") proto.RegisterType((*StoreDescriptor)(nil), "cockroach.roachpb.StoreDescriptor") @@ -835,98 +920,110 @@ func init() { func init() { proto.RegisterFile("roachpb/metadata.proto", fileDescriptor_ecd7f5cc803f82c8) } var fileDescriptor_ecd7f5cc803f82c8 = []byte{ - // 1455 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x56, 0xcf, 0x6f, 0x1b, 0xc5, - 0x17, 0xf7, 0xc6, 0xeb, 0xd8, 0x7e, 0xf9, 0xe5, 0x8c, 0xbe, 0xdf, 0xd6, 0xf2, 0xf7, 0x8b, 0xed, - 0x1a, 0x2a, 0xd2, 0x82, 0x92, 0x34, 0x28, 0xaa, 0x1a, 0x28, 0x52, 0x9c, 0xa4, 0xc1, 0x34, 0x71, - 0xc2, 0xc6, 0x2d, 0x12, 0x97, 0xd5, 0x64, 0x77, 0x70, 0x96, 0xae, 0x77, 0xdd, 0xd9, 0x71, 0x5a, - 0xdf, 0x11, 0x20, 0x21, 0x24, 0x8e, 0x48, 0x5c, 0x8a, 0x10, 0x07, 0xfe, 0x03, 0xfe, 0x84, 0x1e, - 0x7b, 0xec, 0x29, 0x82, 0x54, 0x42, 0x9c, 0x39, 0xe6, 0x80, 0xd0, 0xbc, 0x9d, 0xfd, 0xe1, 0x34, - 0x85, 0xb6, 0x48, 0xdc, 0x66, 0x3f, 0xef, 0xf3, 0x79, 0xfb, 0xe6, 0xcd, 0x7b, 0x6f, 0x06, 0xce, - 0x71, 0x9f, 0x5a, 0x07, 0xfd, 0xfd, 0x85, 0x1e, 0x13, 0xd4, 0xa6, 0x82, 0xce, 0xf7, 0xb9, 0x2f, - 0x7c, 0x32, 0x6b, 0xf9, 0xd6, 0x1d, 0xb4, 0xcd, 0x2b, 0x46, 0xa5, 0x32, 0x10, 0x8e, 0xbb, 0x30, - 0xf0, 0x38, 0x0b, 0x7c, 0xf7, 0x90, 0xd9, 0x26, 0xb5, 0x6d, 0x1e, 0xd2, 0x2b, 0x65, 0xb4, 0x1d, - 0xb8, 0xd6, 0x82, 0x70, 0x7a, 0x2c, 0x10, 0xb4, 0xd7, 0x57, 0x96, 0xff, 0x74, 0xfd, 0xae, 0x8f, - 0xcb, 0x05, 0xb9, 0x0a, 0xd1, 0xc6, 0x1a, 0xc0, 0xaa, 0x10, 0xdc, 0xd9, 0x1f, 0x08, 0x16, 0x90, - 0x37, 0x20, 0x47, 0x85, 0xe0, 0x41, 0x59, 0xab, 0x67, 0xe7, 0x8a, 0xcd, 0xff, 0xfe, 0x7e, 0x54, - 0x9b, 0x1d, 0xd2, 0x9e, 0xbb, 0xd2, 0x40, 0xf8, 0xcd, 0x8f, 0x5d, 0xff, 0x5e, 0xc3, 0x08, 0x39, - 0x2b, 0x85, 0x6f, 0x1e, 0xd4, 0x32, 0xbf, 0x3d, 0xa8, 0x69, 0x8d, 0xcf, 0x35, 0x98, 0x35, 0x58, - 0xdf, 0x75, 0x2c, 0x2a, 0x1c, 0xdf, 0xeb, 0x50, 0xde, 0x65, 0x82, 0x5c, 0x81, 0xbc, 0xe7, 0xdb, - 0xcc, 0x74, 0xec, 0xb2, 0x56, 0xd7, 0xe6, 0x72, 0xcd, 0xf2, 0xc3, 0xa3, 0x5a, 0xe6, 0xf8, 0xa8, - 0x36, 0xde, 0xf6, 0x6d, 0xd6, 0x5a, 0x3f, 0x89, 0x57, 0xc6, 0xb8, 0x24, 0xb6, 0x6c, 0xb2, 0x0c, - 0x85, 0x40, 0xf8, 0x1c, 0x35, 0x63, 0xa8, 0xa9, 0x28, 0x4d, 0x7e, 0x4f, 0xe2, 0x28, 0x8a, 0x96, - 0x46, 0x1e, 0xb9, 0x2d, 0x3b, 0x15, 0xc9, 0x1f, 0x49, 0x24, 0xeb, 0x2c, 0xb0, 0xb8, 0xd3, 0x17, - 0x3e, 0xff, 0xf7, 0x22, 0x21, 0xd7, 0x01, 0x78, 0xf8, 0x7b, 0x29, 0xcc, 0xa2, 0xb0, 0xaa, 0x84, - 0x45, 0x15, 0x18, 0x4a, 0x93, 0x0f, 0xa3, 0xa8, 0x14, 0x2d, 0x9b, 0x2c, 0x81, 0x2e, 0x86, 0x7d, - 0x56, 0xd6, 0xeb, 0xda, 0xdc, 0xf4, 0x52, 0x75, 0xfe, 0xa9, 0xb3, 0x9f, 0x57, 0xb2, 0xce, 0xb0, - 0xcf, 0x0c, 0xe4, 0xae, 0x4c, 0xca, 0xcd, 0xff, 0xf4, 0xa0, 0xa6, 0x61, 0x02, 0xbe, 0xd4, 0x60, - 0x32, 0x72, 0x6d, 0x33, 0x4f, 0xc8, 0x8d, 0x70, 0xea, 0x75, 0xe3, 0xcd, 0x67, 0x93, 0x8d, 0x18, - 0x12, 0x0f, 0x37, 0xa2, 0x96, 0x46, 0x1e, 0xb9, 0x2d, 0x9b, 0xac, 0x43, 0x5e, 0x85, 0x85, 0xdb, - 0x9f, 0x58, 0x7a, 0xed, 0xd9, 0xc1, 0x24, 0x99, 0x6e, 0xea, 0xd2, 0xb7, 0x11, 0x49, 0x1b, 0x9f, - 0xe9, 0x30, 0x83, 0xae, 0x53, 0x87, 0xf1, 0x92, 0x01, 0x5d, 0x84, 0x62, 0x20, 0x28, 0x17, 0xe6, - 0x1d, 0x36, 0xc4, 0x90, 0x26, 0x9b, 0x85, 0x93, 0xa3, 0x9a, 0x6e, 0xdc, 0x64, 0x43, 0xa3, 0x80, - 0xa6, 0x9b, 0x6c, 0x48, 0x2e, 0x40, 0x9e, 0x79, 0x36, 0x92, 0xb2, 0xa7, 0x48, 0xe3, 0xcc, 0xb3, - 0x25, 0xe5, 0x43, 0x98, 0x75, 0x3c, 0xc1, 0xb8, 0x47, 0x5d, 0x53, 0x05, 0x1a, 0x94, 0xf5, 0x7a, - 0xf6, 0x05, 0x37, 0x59, 0x8a, 0x9c, 0x28, 0x42, 0x40, 0xde, 0x87, 0x19, 0x8f, 0xdd, 0x17, 0x66, - 0xaa, 0x02, 0x72, 0x58, 0x01, 0x0d, 0xb5, 0xc1, 0xa9, 0x36, 0xbb, 0x2f, 0x9e, 0x51, 0x05, 0x53, - 0x5e, 0xca, 0x66, 0x93, 0xab, 0x00, 0x5d, 0xe6, 0x31, 0x8e, 0x0d, 0x55, 0x1e, 0xc7, 0x3c, 0x9d, - 0x97, 0x6e, 0x4e, 0x8e, 0x6a, 0x61, 0x4a, 0x37, 0x63, 0xb3, 0x91, 0xa2, 0x92, 0xf7, 0xa0, 0x6e, - 0xb3, 0x3e, 0x67, 0x16, 0x15, 0xcc, 0x36, 0x13, 0x83, 0x69, 0xf9, 0xbd, 0x3e, 0xe5, 0x74, 0xdf, - 0x65, 0xe5, 0x42, 0x5d, 0x9b, 0x2b, 0x18, 0xd5, 0x84, 0x97, 0xf8, 0x5a, 0x8b, 0x59, 0xe4, 0x1d, - 0x80, 0x40, 0x38, 0xd6, 0x9d, 0xa1, 0xb9, 0xef, 0x88, 0x72, 0x1e, 0xab, 0xe0, 0x95, 0x54, 0x82, - 0xe4, 0xa4, 0x99, 0x3f, 0x70, 0xad, 0xf9, 0x4e, 0x34, 0x69, 0x8c, 0x62, 0x28, 0x68, 0x3a, 0x62, - 0xa4, 0x2c, 0x33, 0x8d, 0x1f, 0x34, 0x98, 0xd8, 0x65, 0xdc, 0x62, 0x9e, 0x70, 0x5c, 0x16, 0x90, - 0x73, 0x90, 0xed, 0x5f, 0x59, 0xc4, 0xf3, 0xd7, 0x54, 0x3e, 0x25, 0x80, 0xf8, 0xd2, 0x32, 0x9e, - 0x6f, 0x82, 0x2f, 0x2d, 0x23, 0xbe, 0xbc, 0x88, 0x47, 0x9a, 0xe0, 0xcb, 0x21, 0xff, 0xea, 0x32, - 0xf6, 0x4b, 0x82, 0x5f, 0x0d, 0xf9, 0xd7, 0x16, 0x31, 0xfd, 0x09, 0x7e, 0x6d, 0x91, 0x94, 0x41, - 0xef, 0x6f, 0xd3, 0xfb, 0x98, 0xd0, 0xc8, 0x80, 0xc8, 0x8a, 0x2e, 0xe3, 0x6d, 0x9c, 0x64, 0x61, - 0x0a, 0x9b, 0x7a, 0x8d, 0xf6, 0xa9, 0xe5, 0x88, 0x21, 0xa9, 0x43, 0xc1, 0x52, 0x6b, 0x55, 0xae, - 0xa1, 0x2a, 0x46, 0x49, 0x03, 0x8a, 0xf4, 0x90, 0x3a, 0x2e, 0xa6, 0x76, 0x2c, 0x45, 0x49, 0x60, - 0xf9, 0xdf, 0x41, 0xc0, 0x6c, 0xcc, 0x7c, 0x64, 0x46, 0x84, 0x5c, 0x82, 0x29, 0xd7, 0xef, 0x3a, - 0x16, 0x75, 0xcd, 0xfd, 0xa1, 0x60, 0x41, 0xb9, 0x98, 0xa2, 0x4c, 0x2a, 0x53, 0x53, 0x5a, 0xc8, - 0x45, 0x98, 0x08, 0x3b, 0xc7, 0xf2, 0x07, 0x9e, 0x50, 0xd3, 0x25, 0x24, 0x02, 0x1a, 0xd6, 0x24, - 0x2e, 0x69, 0x2e, 0xa3, 0x41, 0x44, 0xd3, 0xd3, 0x34, 0x34, 0x84, 0xb4, 0x25, 0x20, 0x77, 0x07, - 0x8c, 0x3b, 0x2c, 0x30, 0xfb, 0x8c, 0x9b, 0x01, 0xb3, 0x7c, 0xcf, 0x2e, 0x43, 0x2a, 0x31, 0x25, - 0x65, 0xdf, 0x65, 0x7c, 0x0f, 0xad, 0x64, 0x11, 0x66, 0xef, 0x71, 0x47, 0x8c, 0x4a, 0xd2, 0x49, - 0x9e, 0x09, 0xcd, 0x89, 0x62, 0x17, 0x66, 0x71, 0x5b, 0x28, 0x88, 0x26, 0xca, 0x38, 0xd6, 0xd2, - 0x59, 0xe3, 0x2d, 0x55, 0x23, 0x91, 0x47, 0x94, 0xef, 0x32, 0xae, 0x7a, 0x83, 0x18, 0x40, 0x52, - 0x31, 0x44, 0x2e, 0xf3, 0x2f, 0xe0, 0xb2, 0x14, 0x07, 0xa9, 0x7c, 0xaa, 0xc3, 0xff, 0x55, 0x87, - 0x69, 0x79, 0x0d, 0xfc, 0xb3, 0x9b, 0xe3, 0x5d, 0xc8, 0xcb, 0xfb, 0x98, 0x05, 0x81, 0x9a, 0x9c, - 0xd5, 0xd3, 0x3d, 0x73, 0x2b, 0xbe, 0xb9, 0x57, 0x6d, 0x3b, 0x9e, 0x99, 0x4a, 0x44, 0xae, 0x45, - 0x77, 0x70, 0xf6, 0xa9, 0x8e, 0x8b, 0xb6, 0x94, 0xdc, 0xd8, 0x4a, 0x1c, 0x2a, 0xc8, 0x75, 0x28, - 0xb8, 0xbe, 0x45, 0x5d, 0x59, 0xab, 0x3a, 0xaa, 0xff, 0x77, 0x86, 0x7a, 0x4b, 0x51, 0xa2, 0x42, - 0x8e, 0x24, 0xe4, 0x06, 0x4c, 0xed, 0x31, 0x7e, 0xc8, 0xf8, 0x6d, 0xc6, 0x03, 0x39, 0x76, 0x72, - 0xe8, 0xa3, 0x72, 0x86, 0x0f, 0xc5, 0x50, 0x2e, 0x46, 0x65, 0xe4, 0x02, 0x14, 0xf7, 0x07, 0x8e, - 0x6b, 0x9b, 0x82, 0x76, 0xf1, 0xac, 0x8b, 0xd1, 0xaf, 0x10, 0xee, 0xd0, 0x2e, 0x79, 0x55, 0xce, - 0x16, 0xca, 0xe5, 0x88, 0xa2, 0xe1, 0x6c, 0x89, 0x9b, 0x46, 0xe1, 0xab, 0x82, 0xec, 0x41, 0x29, - 0x8a, 0xcd, 0x8c, 0x52, 0x5a, 0xc0, 0x39, 0xdd, 0xf8, 0x8b, 0x6d, 0xad, 0x86, 0xcc, 0xa8, 0x7c, - 0xdc, 0x51, 0x98, 0xbc, 0x0e, 0x93, 0x96, 0x3b, 0x08, 0x04, 0xe3, 0xa6, 0x47, 0x7b, 0x0c, 0xdb, - 0x2d, 0x8a, 0x6f, 0x42, 0x59, 0xda, 0xb4, 0xc7, 0xc8, 0x1e, 0x4c, 0x04, 0x77, 0xdd, 0xf8, 0xc7, - 0xf0, 0x5c, 0x67, 0x49, 0x54, 0x79, 0xc0, 0xde, 0x07, 0x5b, 0xea, 0x8f, 0x06, 0x04, 0x77, 0x5d, - 0xb5, 0x5e, 0xd1, 0xf1, 0x92, 0xfe, 0x56, 0x83, 0x99, 0x53, 0xe1, 0xa6, 0xcb, 0x46, 0x7b, 0x99, - 0xb2, 0x69, 0xca, 0x39, 0xa2, 0x92, 0x25, 0x1c, 0xc6, 0x55, 0xf1, 0x9d, 0x3f, 0x23, 0x53, 0x1d, - 0x87, 0xf1, 0x64, 0xc0, 0x84, 0x1a, 0x89, 0xa9, 0xe8, 0x3e, 0x1d, 0x83, 0x19, 0x9c, 0x81, 0xa3, - 0x97, 0x76, 0xfc, 0x1c, 0xd2, 0x9e, 0xff, 0x39, 0x14, 0xd7, 0xf2, 0xd8, 0x0b, 0xd7, 0xf2, 0xdb, - 0xa0, 0xcb, 0x86, 0x52, 0x5d, 0x70, 0xe1, 0x0c, 0xe5, 0x68, 0xab, 0x46, 0x43, 0x55, 0x8a, 0x48, - 0x33, 0x35, 0xb4, 0xc3, 0x46, 0xa8, 0x9f, 0xe1, 0x60, 0x64, 0xd0, 0x9f, 0x1e, 0xeb, 0x8d, 0xaf, - 0x34, 0x98, 0x55, 0x69, 0xa0, 0x76, 0x7c, 0xc7, 0xbf, 0x64, 0x22, 0x56, 0xa1, 0x10, 0x3f, 0x35, - 0xc6, 0xb0, 0x84, 0x6b, 0xcf, 0x7e, 0x6a, 0xe0, 0xc3, 0x2d, 0x8a, 0x27, 0x92, 0x35, 0x5a, 0x50, - 0x88, 0x6a, 0x86, 0xbc, 0x05, 0x39, 0x79, 0xc6, 0xe1, 0x3b, 0xfd, 0x6f, 0x0f, 0x39, 0xe4, 0xa6, - 0x5e, 0xc9, 0xeb, 0xa0, 0x4b, 0xb3, 0xbc, 0x25, 0xe5, 0x43, 0x49, 0x4b, 0xb5, 0x80, 0x04, 0x48, - 0x05, 0x72, 0x87, 0xd4, 0x1d, 0x84, 0xb7, 0x59, 0x64, 0x09, 0xa1, 0x94, 0x97, 0x1f, 0x35, 0xc8, - 0x47, 0x2d, 0x7f, 0x19, 0x8a, 0x3d, 0xfa, 0x89, 0xcf, 0xcd, 0x43, 0xea, 0xaa, 0xbc, 0x4c, 0xa9, - 0xbc, 0xe4, 0xb6, 0xa5, 0xc1, 0x28, 0xa0, 0xfd, 0x36, 0x75, 0x91, 0xeb, 0x78, 0x8a, 0x3b, 0x76, - 0x8a, 0x2b, 0x0d, 0x46, 0x01, 0xed, 0x92, 0x5b, 0x81, 0x5c, 0x9f, 0x0a, 0xeb, 0x60, 0xe4, 0xb2, - 0x0b, 0x21, 0x79, 0x33, 0x47, 0x4f, 0xb0, 0x91, 0x4b, 0x2e, 0x46, 0x93, 0x58, 0x2f, 0x7f, 0xa7, - 0xc1, 0x44, 0xea, 0xe9, 0x4c, 0xa6, 0x01, 0x6e, 0xef, 0x74, 0x36, 0x0c, 0xf3, 0xc6, 0xad, 0xad, - 0xad, 0x52, 0x86, 0x10, 0x98, 0x0e, 0xbf, 0x5b, 0xed, 0xb5, 0x9d, 0xed, 0x56, 0x7b, 0xb3, 0x34, - 0x96, 0x60, 0x3b, 0xb7, 0x3a, 0x9b, 0x3b, 0x12, 0xcb, 0x92, 0x0a, 0x9c, 0x0b, 0xb1, 0xf5, 0x8d, - 0xed, 0x9d, 0x4e, 0xab, 0xbd, 0x69, 0x6e, 0x6d, 0xac, 0x1a, 0xed, 0x0d, 0xa3, 0xa4, 0x93, 0x09, - 0xc8, 0x47, 0x1f, 0x1a, 0x99, 0x82, 0x62, 0x7b, 0xa7, 0x6d, 0x22, 0xb9, 0x94, 0x23, 0xff, 0x87, - 0xf2, 0x29, 0x5d, 0x62, 0x1d, 0xaf, 0xe8, 0x5f, 0x7c, 0x5f, 0xcd, 0x34, 0x2f, 0x3d, 0xfc, 0xa5, - 0x9a, 0x79, 0x78, 0x5c, 0xd5, 0x1e, 0x1d, 0x57, 0xb5, 0xc7, 0xc7, 0x55, 0xed, 0xe7, 0xe3, 0xaa, - 0xf6, 0xf5, 0x93, 0x6a, 0xe6, 0xd1, 0x93, 0x6a, 0xe6, 0xf1, 0x93, 0x6a, 0xe6, 0xa3, 0xbc, 0x3a, - 0xdc, 0x3f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xbd, 0xf7, 0xb3, 0x07, 0x2d, 0x0e, 0x00, 0x00, + // 1643 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0xcf, 0x6f, 0x23, 0x49, + 0x15, 0x76, 0xc7, 0xed, 0xb8, 0xfd, 0x92, 0x4c, 0x92, 0x5a, 0x76, 0xd6, 0x32, 0xac, 0xed, 0x31, + 0x2c, 0xcc, 0x2e, 0x28, 0x93, 0x0d, 0x8a, 0x46, 0x13, 0x58, 0xa4, 0x38, 0x99, 0x99, 0x35, 0x9b, + 0xb1, 0x43, 0xc7, 0x33, 0x48, 0x7b, 0x69, 0x55, 0xba, 0x2b, 0x4e, 0x33, 0xed, 0xee, 0x9e, 0xea, + 0x72, 0x76, 0xfc, 0x0f, 0x00, 0x12, 0x42, 0xe2, 0x88, 0xc4, 0x65, 0x10, 0xe2, 0xc0, 0x9d, 0x03, + 0x7f, 0xc2, 0x9c, 0x60, 0x8f, 0x7b, 0x8a, 0x20, 0x23, 0x21, 0xce, 0x1c, 0xe7, 0x80, 0x50, 0xbd, + 0xaa, 0xfe, 0xe1, 0x8c, 0x03, 0x3b, 0x83, 0xc4, 0xad, 0xfc, 0xbd, 0xef, 0x7b, 0xfd, 0xea, 0xd5, + 0xab, 0x57, 0xcf, 0x70, 0x9d, 0x47, 0xd4, 0x3d, 0x8d, 0x8f, 0x6f, 0x8d, 0x99, 0xa0, 0x1e, 0x15, + 0x74, 0x23, 0xe6, 0x91, 0x88, 0xc8, 0xba, 0x1b, 0xb9, 0x8f, 0xd1, 0xb6, 0xa1, 0x19, 0x8d, 0xc6, + 0x44, 0xf8, 0xc1, 0xad, 0x49, 0xc8, 0x59, 0x12, 0x05, 0x67, 0xcc, 0x73, 0xa8, 0xe7, 0x71, 0x45, + 0x6f, 0xd4, 0xd1, 0x76, 0x1a, 0xb8, 0xb7, 0x84, 0x3f, 0x66, 0x89, 0xa0, 0xe3, 0x58, 0x5b, 0xbe, + 0x32, 0x8a, 0x46, 0x11, 0x2e, 0x6f, 0xc9, 0x95, 0x42, 0x3b, 0x7b, 0x00, 0xbb, 0x42, 0x70, 0xff, + 0x78, 0x22, 0x58, 0x42, 0xbe, 0x0d, 0x15, 0x2a, 0x04, 0x4f, 0xea, 0x46, 0xbb, 0x7c, 0xb3, 0xd6, + 0x7d, 0xfb, 0x9f, 0xe7, 0xad, 0xf5, 0x29, 0x1d, 0x07, 0x3b, 0x1d, 0x84, 0xbf, 0x73, 0x12, 0x44, + 0x9f, 0x75, 0x6c, 0xc5, 0xd9, 0xb1, 0x7e, 0xfd, 0xac, 0x55, 0xfa, 0xc7, 0xb3, 0x96, 0xd1, 0xf9, + 0x99, 0x01, 0xeb, 0x36, 0x8b, 0x03, 0xdf, 0xa5, 0xc2, 0x8f, 0xc2, 0x21, 0xe5, 0x23, 0x26, 0xc8, + 0x87, 0x50, 0x0d, 0x23, 0x8f, 0x39, 0xbe, 0x57, 0x37, 0xda, 0xc6, 0xcd, 0x4a, 0xb7, 0xfe, 0xfc, + 0xbc, 0x55, 0xba, 0x38, 0x6f, 0x2d, 0xf6, 0x23, 0x8f, 0xf5, 0xf6, 0x5f, 0x66, 0x2b, 0x7b, 0x51, + 0x12, 0x7b, 0x1e, 0xd9, 0x06, 0x2b, 0x11, 0x11, 0x47, 0xcd, 0x02, 0x6a, 0x1a, 0x5a, 0x53, 0x3d, + 0x92, 0x38, 0x8a, 0xd2, 0xa5, 0x5d, 0x45, 0x6e, 0xcf, 0x2b, 0x44, 0xf2, 0xaf, 0x3c, 0x92, 0x7d, + 0x96, 0xb8, 0xdc, 0x8f, 0x45, 0xc4, 0xff, 0x7f, 0x91, 0x90, 0x8f, 0x00, 0xb8, 0xfa, 0xbc, 0x14, + 0x96, 0x51, 0xd8, 0xd4, 0xc2, 0x9a, 0x0e, 0x0c, 0xa5, 0xf9, 0x0f, 0xbb, 0xa6, 0x15, 0x3d, 0x8f, + 0x6c, 0x81, 0x29, 0xa6, 0x31, 0xab, 0x9b, 0x6d, 0xe3, 0xe6, 0xb5, 0xad, 0xe6, 0xc6, 0x2b, 0x67, + 0xbf, 0xa1, 0x65, 0xc3, 0x69, 0xcc, 0x6c, 0xe4, 0xee, 0x2c, 0xcb, 0xcd, 0xff, 0xe9, 0x59, 0xcb, + 0xc0, 0x04, 0xfc, 0xc2, 0x80, 0xe5, 0xd4, 0xb5, 0xc7, 0x42, 0x21, 0x37, 0xc2, 0x69, 0x38, 0xca, + 0x36, 0x5f, 0xce, 0x37, 0x62, 0x4b, 0x5c, 0x6d, 0x44, 0x2f, 0xed, 0x2a, 0x72, 0x7b, 0x1e, 0xd9, + 0x87, 0xaa, 0x0e, 0x0b, 0xb7, 0xbf, 0xb4, 0xf5, 0x8d, 0xab, 0x83, 0xc9, 0x33, 0xdd, 0x35, 0xa5, + 0x6f, 0x3b, 0x95, 0x76, 0x7e, 0x6a, 0xc2, 0x2a, 0xba, 0x2e, 0x1c, 0xc6, 0x1b, 0x06, 0xf4, 0x1e, + 0xd4, 0x12, 0x41, 0xb9, 0x70, 0x1e, 0xb3, 0x29, 0x86, 0xb4, 0xdc, 0xb5, 0x5e, 0x9e, 0xb7, 0x4c, + 0xfb, 0x13, 0x36, 0xb5, 0x2d, 0x34, 0x7d, 0xc2, 0xa6, 0xe4, 0x06, 0x54, 0x59, 0xe8, 0x21, 0xa9, + 0x7c, 0x89, 0xb4, 0xc8, 0x42, 0x4f, 0x52, 0x7e, 0x0c, 0xeb, 0x7e, 0x28, 0x18, 0x0f, 0x69, 0xe0, + 0xe8, 0x40, 0x93, 0xba, 0xd9, 0x2e, 0xbf, 0xe6, 0x26, 0xd7, 0x52, 0x27, 0x9a, 0x90, 0x90, 0x1f, + 0xc2, 0x6a, 0xc8, 0x9e, 0x0a, 0xa7, 0x50, 0x01, 0x15, 0xac, 0x80, 0x8e, 0xde, 0xe0, 0x4a, 0x9f, + 0x3d, 0x15, 0x57, 0x54, 0xc1, 0x4a, 0x58, 0xb0, 0x79, 0xe4, 0x36, 0xc0, 0x88, 0x85, 0x8c, 0xe3, + 0x85, 0xaa, 0x2f, 0x62, 0x9e, 0xde, 0x91, 0x6e, 0x5e, 0x9e, 0xb7, 0x54, 0x4a, 0xef, 0x67, 0x66, + 0xbb, 0x40, 0x25, 0x1f, 0x43, 0xdb, 0x63, 0x31, 0x67, 0x2e, 0x15, 0xcc, 0x73, 0x72, 0x83, 0xe3, + 0x46, 0xe3, 0x98, 0x72, 0x7a, 0x1c, 0xb0, 0xba, 0xd5, 0x36, 0x6e, 0x5a, 0x76, 0x33, 0xe7, 0xe5, + 0xbe, 0xf6, 0x32, 0x16, 0xf9, 0x3e, 0x40, 0x22, 0x7c, 0xf7, 0xf1, 0xd4, 0x39, 0xf6, 0x45, 0xbd, + 0x8a, 0x55, 0xf0, 0x6e, 0x21, 0x41, 0xb2, 0xd3, 0x6c, 0x9c, 0x06, 0xee, 0xc6, 0x30, 0xed, 0x34, + 0x76, 0x4d, 0x09, 0xba, 0xbe, 0x98, 0x29, 0xcb, 0x52, 0xe7, 0xf7, 0x06, 0x2c, 0x1d, 0x32, 0xee, + 0xb2, 0x50, 0xf8, 0x01, 0x4b, 0xc8, 0x75, 0x28, 0xc7, 0x1f, 0x6e, 0xe2, 0xf9, 0x1b, 0x3a, 0x9f, + 0x12, 0x40, 0x7c, 0x6b, 0x1b, 0xcf, 0x37, 0xc7, 0xb7, 0xb6, 0x11, 0xdf, 0xde, 0xc4, 0x23, 0xcd, + 0xf1, 0x6d, 0xc5, 0xbf, 0xbd, 0x8d, 0xf7, 0x25, 0xc7, 0x6f, 0x2b, 0xfe, 0x9d, 0x4d, 0x4c, 0x7f, + 0x8e, 0xdf, 0xd9, 0x24, 0x75, 0x30, 0xe3, 0x07, 0xf4, 0x29, 0x26, 0x34, 0x35, 0x20, 0xb2, 0x63, + 0xca, 0x78, 0x3b, 0x2f, 0xcb, 0xb0, 0x82, 0x97, 0x7a, 0x8f, 0xc6, 0xd4, 0xf5, 0xc5, 0x94, 0xb4, + 0xc1, 0x72, 0xf5, 0x5a, 0x97, 0xab, 0x52, 0x65, 0x28, 0xe9, 0x40, 0x8d, 0x9e, 0x51, 0x3f, 0xc0, + 0xd4, 0x2e, 0x14, 0x28, 0x39, 0x2c, 0xbf, 0x3b, 0x49, 0x98, 0x87, 0x99, 0x4f, 0xcd, 0x88, 0x90, + 0xf7, 0x61, 0x25, 0x88, 0x46, 0xbe, 0x4b, 0x03, 0xe7, 0x78, 0x2a, 0x58, 0x52, 0xaf, 0x15, 0x28, + 0xcb, 0xda, 0xd4, 0x95, 0x16, 0xf2, 0x1e, 0x2c, 0xa9, 0x9b, 0xe3, 0x46, 0x93, 0x50, 0xe8, 0xee, + 0xa2, 0x88, 0x80, 0x86, 0x3d, 0x89, 0x4b, 0x5a, 0xc0, 0x68, 0x92, 0xd2, 0xcc, 0x22, 0x0d, 0x0d, + 0x8a, 0xb6, 0x05, 0xe4, 0xc9, 0x84, 0x71, 0x9f, 0x25, 0x4e, 0xcc, 0xb8, 0x93, 0x30, 0x37, 0x0a, + 0xbd, 0x3a, 0x14, 0x12, 0xb3, 0xa6, 0xed, 0x87, 0x8c, 0x1f, 0xa1, 0x95, 0x6c, 0xc2, 0xfa, 0x67, + 0xdc, 0x17, 0xb3, 0x92, 0x62, 0x92, 0x57, 0x95, 0x39, 0x57, 0x1c, 0xc2, 0x3a, 0x6e, 0x0b, 0x05, + 0x69, 0x47, 0x59, 0xc4, 0x5a, 0x9a, 0xd7, 0xde, 0x0a, 0x35, 0x92, 0x7a, 0x44, 0xf9, 0x21, 0xe3, + 0xfa, 0x6e, 0x10, 0x1b, 0x48, 0x21, 0x86, 0xd4, 0x65, 0xf5, 0x35, 0x5c, 0xae, 0x65, 0x41, 0x6a, + 0x9f, 0xfa, 0xf0, 0xff, 0x68, 0xc0, 0x2a, 0x1e, 0xfe, 0x21, 0x8f, 0x62, 0xc6, 0x85, 0xcf, 0x12, + 0x79, 0xb8, 0x2c, 0x74, 0xf9, 0x34, 0x16, 0x4c, 0xb5, 0x2b, 0x2b, 0x3d, 0xdc, 0x0c, 0x26, 0x37, + 0xa0, 0xc6, 0x19, 0xf5, 0x9c, 0x28, 0x0c, 0x54, 0x6b, 0x4a, 0x39, 0x96, 0x84, 0x07, 0x61, 0x30, + 0x25, 0x9f, 0xc2, 0xdb, 0x27, 0x7e, 0xc0, 0x1c, 0xf5, 0xa6, 0xc4, 0x99, 0x7f, 0x3c, 0xc4, 0xa5, + 0xad, 0x6f, 0xce, 0x89, 0xfb, 0x9e, 0x1f, 0xb0, 0x4b, 0xd1, 0xd8, 0x6f, 0x9d, 0xbc, 0x0a, 0x76, + 0xfe, 0x62, 0xc0, 0x5b, 0x73, 0xc8, 0x58, 0xeb, 0x54, 0x9c, 0x62, 0xd4, 0xb5, 0xac, 0xd6, 0xa9, + 0x38, 0x25, 0xef, 0x42, 0xf5, 0x24, 0x71, 0xf0, 0xa5, 0x59, 0x28, 0x18, 0x17, 0x4f, 0x12, 0xf9, + 0xae, 0x90, 0x6f, 0xc1, 0xf2, 0x71, 0x10, 0xb9, 0x8f, 0x1d, 0x8f, 0x9d, 0xf9, 0x2e, 0xc3, 0x18, + 0x53, 0xce, 0x12, 0x5a, 0xf6, 0xd1, 0x20, 0x2b, 0x6d, 0x2c, 0x6b, 0xc9, 0x89, 0x23, 0x5f, 0x57, + 0x5a, 0xca, 0x03, 0x34, 0x1c, 0x4a, 0x5c, 0x96, 0xb8, 0xa2, 0x45, 0xb1, 0x6c, 0x31, 0x09, 0x56, + 0x4c, 0x4a, 0x5c, 0x46, 0xd3, 0x40, 0x59, 0xf4, 0x41, 0xfc, 0xdd, 0x84, 0x6b, 0xf2, 0x3d, 0xfe, + 0xdf, 0x9e, 0xf0, 0x1f, 0x40, 0x55, 0x0e, 0x46, 0x2c, 0x49, 0xf4, 0x13, 0xd6, 0xbc, 0xdc, 0xbc, + 0x1e, 0x66, 0x23, 0xd4, 0xae, 0xe7, 0x65, 0x8f, 0x97, 0x16, 0x91, 0x3b, 0xe9, 0x30, 0x54, 0x7e, + 0xa5, 0xf5, 0xa5, 0x67, 0x94, 0x8f, 0x4e, 0x5a, 0xac, 0x14, 0xe4, 0x23, 0xb0, 0x82, 0xc8, 0xa5, + 0x81, 0x6c, 0x1a, 0x26, 0xaa, 0xbf, 0x3a, 0x47, 0x7d, 0xa0, 0x29, 0x69, 0xb5, 0xa4, 0x12, 0x72, + 0x0f, 0x56, 0x8e, 0x18, 0x3f, 0x63, 0xfc, 0x11, 0xe3, 0x89, 0xec, 0xff, 0x15, 0xf4, 0xd1, 0x98, + 0xe3, 0x43, 0x33, 0xb4, 0x8b, 0x59, 0x99, 0x2c, 0xcc, 0xe3, 0x89, 0x1f, 0x78, 0x8e, 0xa0, 0x23, + 0xbc, 0x74, 0x69, 0xd2, 0x2d, 0x84, 0x87, 0x74, 0x44, 0xbe, 0x2e, 0x9b, 0x3c, 0xe5, 0xf2, 0xad, + 0xa0, 0xaa, 0xc9, 0x67, 0xdd, 0x4b, 0xe3, 0xbb, 0x82, 0x1c, 0xc1, 0x5a, 0x1a, 0x9b, 0x93, 0xa6, + 0xd4, 0xc2, 0x07, 0xb3, 0xf3, 0x1f, 0xb6, 0xb5, 0xab, 0x98, 0xe9, 0x3d, 0x0e, 0x66, 0x61, 0x59, + 0x65, 0x6e, 0x30, 0x49, 0x04, 0xe3, 0x4e, 0x48, 0xc7, 0x0c, 0xfb, 0x5e, 0x56, 0x65, 0xda, 0xd2, + 0xa7, 0x63, 0x46, 0x8e, 0x60, 0x29, 0x79, 0x12, 0x64, 0x1f, 0x86, 0x2f, 0x75, 0x96, 0x44, 0x97, + 0x07, 0x1c, 0xfd, 0xe8, 0x40, 0x7f, 0xd1, 0x86, 0xe4, 0x49, 0xa0, 0xd7, 0x3b, 0x26, 0x4e, 0x4b, + 0xbf, 0x31, 0x60, 0xf5, 0x52, 0xb8, 0xc5, 0xb2, 0x31, 0xde, 0xa4, 0x6c, 0xba, 0xb2, 0xa1, 0xeb, + 0x64, 0x09, 0x9f, 0x71, 0x5d, 0x7c, 0xef, 0xcc, 0xc9, 0xd4, 0xd0, 0x67, 0x3c, 0xef, 0xf4, 0x4a, + 0x23, 0x31, 0x1d, 0xdd, 0x9f, 0x17, 0x74, 0x3f, 0x9a, 0x9d, 0x9e, 0xb2, 0xb9, 0xd4, 0xf8, 0xf2, + 0x73, 0x69, 0x56, 0xcb, 0x0b, 0xaf, 0x5d, 0xcb, 0xdf, 0x03, 0x53, 0x5e, 0x28, 0x7d, 0x0b, 0x6e, + 0xcc, 0x51, 0xce, 0x5e, 0xd5, 0xb4, 0xd3, 0x48, 0x11, 0xe9, 0x16, 0x5e, 0x4f, 0x75, 0x11, 0xda, + 0x73, 0x1c, 0xcc, 0xbc, 0xb8, 0xaf, 0xbc, 0xaf, 0x1f, 0x03, 0x14, 0x1a, 0xa6, 0xba, 0x0a, 0x9d, + 0xab, 0xbc, 0xe4, 0xfd, 0x2f, 0x6d, 0x44, 0xb9, 0xb6, 0xf3, 0x4b, 0x03, 0xd6, 0x75, 0x42, 0xa9, + 0x97, 0x8d, 0x6d, 0x6f, 0x98, 0xd2, 0x5d, 0xb0, 0xb2, 0xe9, 0x71, 0x01, 0x2f, 0x43, 0xeb, 0xea, + 0xe9, 0x11, 0x67, 0xf1, 0xfc, 0x55, 0x50, 0xb2, 0x4e, 0x0f, 0xac, 0xb4, 0xfa, 0xc8, 0x77, 0xa1, + 0x22, 0xab, 0x45, 0xfd, 0xf5, 0xfa, 0xaf, 0xe5, 0xa2, 0xb8, 0x85, 0x3f, 0x3e, 0xfb, 0x60, 0x4a, + 0xb3, 0x1c, 0x7c, 0xe4, 0xec, 0x5b, 0xec, 0xf9, 0x12, 0x20, 0x0d, 0xa8, 0x9c, 0xd1, 0x60, 0x32, + 0xdb, 0xf0, 0x15, 0x54, 0xf0, 0xf2, 0x07, 0x03, 0xaa, 0x69, 0xf3, 0xf8, 0x00, 0x6a, 0x63, 0xfa, + 0x93, 0x88, 0x3b, 0x67, 0x34, 0xd0, 0x79, 0x59, 0xd1, 0x79, 0xa9, 0x3c, 0x90, 0x06, 0xdb, 0x42, + 0xfb, 0x23, 0x1a, 0x20, 0xd7, 0x0f, 0x35, 0x77, 0xe1, 0x12, 0x57, 0x1a, 0x6c, 0x0b, 0xed, 0x92, + 0xdb, 0x80, 0x4a, 0x4c, 0x85, 0x7b, 0x3a, 0x33, 0xbf, 0x28, 0x48, 0x0e, 0x5b, 0xe9, 0x54, 0x3d, + 0x33, 0xb7, 0x64, 0x68, 0x1e, 0xeb, 0x07, 0xbf, 0x35, 0x60, 0xa9, 0xf0, 0x6f, 0x88, 0x5c, 0x03, + 0x78, 0x34, 0x18, 0xde, 0xb5, 0x9d, 0x7b, 0x0f, 0x0f, 0x0e, 0xd6, 0x4a, 0x84, 0xc0, 0x35, 0xf5, + 0xbb, 0xd7, 0xdf, 0x1b, 0x3c, 0xe8, 0xf5, 0xef, 0xaf, 0x2d, 0xe4, 0xd8, 0xe0, 0xe1, 0xf0, 0xfe, + 0x40, 0x62, 0x65, 0xd2, 0x80, 0xeb, 0x0a, 0xdb, 0xbf, 0xfb, 0x60, 0x30, 0xec, 0xf5, 0xef, 0x3b, + 0x07, 0x77, 0x77, 0xed, 0xfe, 0x5d, 0x7b, 0xcd, 0x24, 0x4b, 0x50, 0x4d, 0x7f, 0x18, 0x64, 0x05, + 0x6a, 0xfd, 0x41, 0xdf, 0x41, 0xf2, 0x5a, 0x85, 0x7c, 0x0d, 0xea, 0x97, 0x74, 0xb9, 0x75, 0xb1, + 0x61, 0xfe, 0xfc, 0x77, 0xcd, 0x52, 0xf7, 0xfd, 0xe7, 0x7f, 0x6b, 0x96, 0x9e, 0x5f, 0x34, 0x8d, + 0xcf, 0x2f, 0x9a, 0xc6, 0x17, 0x17, 0x4d, 0xe3, 0xaf, 0x17, 0x4d, 0xe3, 0x57, 0x2f, 0x9a, 0xa5, + 0xcf, 0x5f, 0x34, 0x4b, 0x5f, 0xbc, 0x68, 0x96, 0x3e, 0xad, 0xea, 0xc3, 0xfd, 0x77, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x92, 0xf0, 0x2a, 0x4b, 0x00, 0x10, 0x00, 0x00, } func (this *Attributes) Equal(that interface{}) bool { @@ -1528,6 +1625,105 @@ func (m *StoreCapacity) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *StoreProperties) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *StoreProperties) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *StoreProperties) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.FileStoreProperties != nil { + { + size, err := m.FileStoreProperties.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetadata(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + i-- + if m.ReadOnly { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x10 + i-- + if m.Encrypted { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x8 + return len(dAtA) - i, nil +} + +func (m *FileStoreProperties) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileStoreProperties) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FileStoreProperties) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + i -= len(m.MountOptions) + copy(dAtA[i:], m.MountOptions) + i = encodeVarintMetadata(dAtA, i, uint64(len(m.MountOptions))) + i-- + dAtA[i] = 0x2a + i -= len(m.MountPoint) + copy(dAtA[i:], m.MountPoint) + i = encodeVarintMetadata(dAtA, i, uint64(len(m.MountPoint))) + i-- + dAtA[i] = 0x22 + i -= len(m.BlockDevice) + copy(dAtA[i:], m.BlockDevice) + i = encodeVarintMetadata(dAtA, i, uint64(len(m.BlockDevice))) + i-- + dAtA[i] = 0x1a + i -= len(m.FsType) + copy(dAtA[i:], m.FsType) + i = encodeVarintMetadata(dAtA, i, uint64(len(m.FsType))) + i-- + dAtA[i] = 0x12 + i -= len(m.Path) + copy(dAtA[i:], m.Path) + i = encodeVarintMetadata(dAtA, i, uint64(len(m.Path))) + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + func (m *NodeDescriptor) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -1694,6 +1890,16 @@ func (m *StoreDescriptor) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + { + size, err := m.Properties.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMetadata(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a { size, err := m.Capacity.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -2151,6 +2357,40 @@ func (m *StoreCapacity) Size() (n int) { return n } +func (m *StoreProperties) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += 2 + n += 2 + if m.FileStoreProperties != nil { + l = m.FileStoreProperties.Size() + n += 1 + l + sovMetadata(uint64(l)) + } + return n +} + +func (m *FileStoreProperties) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Path) + n += 1 + l + sovMetadata(uint64(l)) + l = len(m.FsType) + n += 1 + l + sovMetadata(uint64(l)) + l = len(m.BlockDevice) + n += 1 + l + sovMetadata(uint64(l)) + l = len(m.MountPoint) + n += 1 + l + sovMetadata(uint64(l)) + l = len(m.MountOptions) + n += 1 + l + sovMetadata(uint64(l)) + return n +} + func (m *NodeDescriptor) Size() (n int) { if m == nil { return 0 @@ -2208,6 +2448,8 @@ func (m *StoreDescriptor) Size() (n int) { n += 1 + l + sovMetadata(uint64(l)) l = m.Capacity.Size() n += 1 + l + sovMetadata(uint64(l)) + l = m.Properties.Size() + n += 1 + l + sovMetadata(uint64(l)) return n } @@ -3307,6 +3549,342 @@ func (m *StoreCapacity) Unmarshal(dAtA []byte) error { } return nil } +func (m *StoreProperties) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: StoreProperties: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: StoreProperties: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Encrypted", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Encrypted = bool(v != 0) + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ReadOnly", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ReadOnly = bool(v != 0) + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FileStoreProperties", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMetadata + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthMetadata + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.FileStoreProperties == nil { + m.FileStoreProperties = &FileStoreProperties{} + } + if err := m.FileStoreProperties.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMetadata(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthMetadata + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileStoreProperties) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileStoreProperties: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileStoreProperties: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Path", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMetadata + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMetadata + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Path = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FsType", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMetadata + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMetadata + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FsType = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BlockDevice", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMetadata + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMetadata + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BlockDevice = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MountPoint", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMetadata + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMetadata + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.MountPoint = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MountOptions", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMetadata + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMetadata + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.MountOptions = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMetadata(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthMetadata + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *NodeDescriptor) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -3921,6 +4499,39 @@ func (m *StoreDescriptor) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Properties", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMetadata + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMetadata + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthMetadata + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Properties.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipMetadata(dAtA[iNdEx:]) diff --git a/pkg/roachpb/metadata.proto b/pkg/roachpb/metadata.proto index 0372b07de05e..3bb72b2b35bb 100644 --- a/pkg/roachpb/metadata.proto +++ b/pkg/roachpb/metadata.proto @@ -334,6 +334,34 @@ message StoreCapacity { optional Percentiles writes_per_replica = 7 [(gogoproto.nullable) = false]; } +// StoreProperties contains configuration and OS-level details for a storage device. +message StoreProperties { + // encrypted indicates whether the store is encrypted. + optional bool encrypted = 1 [(gogoproto.nullable) = false]; + // read_only indicates whether the store is attached read_only. + optional bool read_only = 2 [(gogoproto.nullable) = false]; + + // disk_properties reports details about the underlying filesystem, + // when the store is supported by a file store. Unset otherwise. + optional FileStoreProperties file_store_properties = 3; +} + +// FileStoreProperties contains configuration and OS-level details for a file store. +message FileStoreProperties { + option (gogoproto.goproto_stringer) = false; + + // path reports the configured filesystem path for the store. + optional string path = 1 [(gogoproto.nullable) = false]; + // fs_type reports the external filesystem type (ufs, ext4, etc), if known. + optional string fs_type = 2 [(gogoproto.nullable) = false]; + // block_device reports which block devices supports the filesystem, if known. + optional string block_device = 3 [(gogoproto.nullable) = false]; + // mount_point reports the mount point of the filesystem, if known. + optional string mount_point = 4 [(gogoproto.nullable) = false]; + // mount_options reports the mount options, if known. + optional string mount_options = 5 [(gogoproto.nullable) = false]; +} + // NodeDescriptor holds details on node physical/network topology. message NodeDescriptor { option (gogoproto.equal) = true; @@ -369,6 +397,7 @@ message StoreDescriptor { optional Attributes attrs = 2 [(gogoproto.nullable) = false]; optional NodeDescriptor node = 3 [(gogoproto.nullable) = false]; optional StoreCapacity capacity = 4 [(gogoproto.nullable) = false]; + optional StoreProperties properties = 5 [(gogoproto.nullable) = false]; } // StoreDeadReplicas holds a storeID and a list of dead replicas on that store. diff --git a/pkg/server/config.go b/pkg/server/config.go index 1927c4b7e7b4..2d0b86acc594 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -539,6 +539,7 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) { if err != nil { return Engines{}, err } + details = append(details, redact.Sprintf("store %d: %+v", i, e.Properties())) engines = append(engines, e) } else { e, err := storage.Open(ctx, @@ -604,6 +605,7 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) { if err != nil { return Engines{}, err } + details = append(details, redact.Sprintf("store %d: %+v", i, eng.Properties())) engines = append(engines, eng) } } diff --git a/pkg/storage/BUILD.bazel b/pkg/storage/BUILD.bazel index bc190272fcdc..1e0093847212 100644 --- a/pkg/storage/BUILD.bazel +++ b/pkg/storage/BUILD.bazel @@ -34,6 +34,7 @@ go_library( "slice_go1.9.go", "sst_iterator.go", "sst_writer.go", + "store_properties.go", "temp_engine.go", "testing_knobs.go", ":gen-resourcelimitreached-stringer", # keep @@ -79,6 +80,7 @@ go_library( "@com_github_cockroachdb_pebble//vfs/atomicfs", "@com_github_cockroachdb_redact//:redact", "@com_github_dustin_go_humanize//:go-humanize", + "@com_github_elastic_gosigar//:gosigar", "@com_github_gogo_protobuf//proto", ], ) diff --git a/pkg/storage/engine.go b/pkg/storage/engine.go index 797d9803591d..97fbd39ecde8 100644 --- a/pkg/storage/engine.go +++ b/pkg/storage/engine.go @@ -743,6 +743,8 @@ type Engine interface { Attrs() roachpb.Attributes // Capacity returns capacity details for the engine's available storage. Capacity() (roachpb.StoreCapacity, error) + // Properties returns the low-level properties for the engine's underlying storage. + Properties() roachpb.StoreProperties // Compact forces compaction over the entire database. Compact() error // Flush causes the engine to write all in-memory data to disk diff --git a/pkg/storage/pebble.go b/pkg/storage/pebble.go index c2962c0933dd..7e4fffa9d49e 100644 --- a/pkg/storage/pebble.go +++ b/pkg/storage/pebble.go @@ -49,7 +49,7 @@ import ( "github.com/cockroachdb/pebble/bloom" "github.com/cockroachdb/pebble/vfs" "github.com/cockroachdb/redact" - "github.com/dustin/go-humanize" + humanize "github.com/dustin/go-humanize" ) const maxSyncDurationFatalOnExceededDefault = true @@ -470,6 +470,7 @@ type Pebble struct { ballastSize int64 maxSize int64 attrs roachpb.Attributes + properties roachpb.StoreProperties // settings must be non-nil if this Pebble instance will be used to write // intents. settings *cluster.Settings @@ -661,6 +662,8 @@ func NewPebble(ctx context.Context, cfg PebbleConfig) (*Pebble, error) { } } + storeProps := computeStoreProperties(ctx, cfg.Dir, cfg.Opts.ReadOnly, env != nil /* encryptionEnabled */) + p := &Pebble{ readOnly: cfg.Opts.ReadOnly, path: cfg.Dir, @@ -669,6 +672,7 @@ func NewPebble(ctx context.Context, cfg PebbleConfig) (*Pebble, error) { ballastSize: cfg.BallastSize, maxSize: cfg.MaxSize, attrs: cfg.Attrs, + properties: storeProps, settings: cfg.Settings, encryption: env, fileRegistry: fileRegistry, @@ -1062,6 +1066,11 @@ func (p *Pebble) Attrs() roachpb.Attributes { return p.attrs } +// Properties implements the Engine interface. +func (p *Pebble) Properties() roachpb.StoreProperties { + return p.properties +} + // Capacity implements the Engine interface. func (p *Pebble) Capacity() (roachpb.StoreCapacity, error) { dir := p.path diff --git a/pkg/storage/store_properties.go b/pkg/storage/store_properties.go new file mode 100644 index 000000000000..86ece2585e46 --- /dev/null +++ b/pkg/storage/store_properties.go @@ -0,0 +1,99 @@ +// Copyright 2021 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package storage + +import ( + "context" + "path/filepath" + + "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/elastic/gosigar" +) + +func computeStoreProperties( + ctx context.Context, dir string, readonly bool, encryptionEnabled bool, +) roachpb.StoreProperties { + props := roachpb.StoreProperties{ + ReadOnly: readonly, + Encrypted: encryptionEnabled, + } + + // In-memory store? + if dir == "" { + return props + } + + fsprops := getFileSystemProperties(ctx, dir) + props.FileStoreProperties = &fsprops + return props +} + +func getFileSystemProperties(ctx context.Context, dir string) roachpb.FileStoreProperties { + fsprops := roachpb.FileStoreProperties{ + Path: dir, + } + + // Find which filesystem supports the store. + + absPath, err := filepath.Abs(dir) + if err != nil { + log.Warningf(ctx, "cannot compute absolute file path for %q: %v", dir, err) + return fsprops + } + + // Alas, only BSD reliably populates "fs" in os.StatFs(), + // so we must find the filesystem manually. + // + // Note that scanning the list of mounts is also + // what linux' df(1) command does. + // + var fslist gosigar.FileSystemList + if err := fslist.Get(); err != nil { + log.Warningf(ctx, "cannot retrieve filesystem list: %v", err) + return fsprops + } + + var fsInfo *gosigar.FileSystem + // We're reading the list of mounts in reverse order: we're assuming + // that mounts are LIFO and can only be stacked, so the best match + // will necessarily be the first filesystem that's a prefix of the + // target directory, when looking from the end of the file. + // + // TODO(ssd): Steven points out that gosigar reads from /etc/mtab on + // linux, which is sometimes managed by the user command 'mount' and + // can sometimes miss entries when `mount -n` is used. It might be + // better to change gosigar to use /proc/mounts instead. + // + // FWIW, we are OK with this for now, since the systems where crdb + // is typically being deployed are well-behaved in that regard: + // Kubernetes mirrors /proc/mount in /etc/mtab. + for i := len(fslist.List) - 1; i >= 0; i-- { + // filepath.Rel can reliably tell us if a path is relative to + // another: if it is not, an error is returned. + _, err := filepath.Rel(fslist.List[i].DirName, absPath) + if err == nil { + fsInfo = &fslist.List[i] + break + } + } + if fsInfo == nil { + // This is surprising!? We're expecting at least a match on the + // root filesystem. Oh well. + return fsprops + } + + fsprops.FsType = fsInfo.SysTypeName + fsprops.BlockDevice = fsInfo.DevName + fsprops.MountPoint = fsInfo.DirName + fsprops.MountOptions = fsInfo.Options + return fsprops +}