Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cli,storage: add emergency ballast #66893

Merged
merged 1 commit into from
Aug 19, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions build/teamcity-support.sh
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ function run_json_test() {
rm -f "${fullfile}"
fi
rm -f "${tmpfile}" artifacts/stripped.txt

# Some unit tests test automatic ballast creation. These ballasts can be
# larger than the maximum artifact size. Remove any artifacts with the
# EMERGENCY_BALLAST filename.
find artifacts -name "EMERGENCY_BALLAST" -delete

tc_end_block "artifacts"

# Make it easier to figure out whether we're exiting because of a test failure
Expand Down
3 changes: 3 additions & 0 deletions pkg/base/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -510,6 +510,9 @@ type StorageConfig struct {
// MaxSize is used for calculating free space and making rebalancing
// decisions. Zero indicates that there is no maximum size.
MaxSize int64
// BallastSize is the amount reserved by a ballast file for manual
// out-of-disk recovery.
BallastSize int64
// Settings instance for cluster-wide knobs.
Settings *cluster.Settings
// UseFileRegistry is true if the file registry is needed (eg: encryption-at-rest).
Expand Down
66 changes: 52 additions & 14 deletions pkg/base/store_spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/cockroachdb/errors"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble"
"github.com/cockroachdb/redact"
humanize "github.com/dustin/go-humanize"
"github.com/spf13/pflag"
)
Expand Down Expand Up @@ -79,7 +80,7 @@ type floatInterval struct {
// NewSizeSpec parses the string passed into a --size flag and returns a
// SizeSpec if it is correctly parsed.
func NewSizeSpec(
value string, bytesRange *intInterval, percentRange *floatInterval,
field redact.SafeString, value string, bytesRange *intInterval, percentRange *floatInterval,
) (SizeSpec, error) {
var size SizeSpec
if fractionRegex.MatchString(value) {
Expand All @@ -93,13 +94,14 @@ func NewSizeSpec(
size.Percent, err = strconv.ParseFloat(factorValue, 64)
size.Percent *= percentFactor
if err != nil {
return SizeSpec{}, fmt.Errorf("could not parse store size (%s) %s", value, err)
return SizeSpec{}, errors.Newf("could not parse %s size (%s) %s", field, value, err)
}
if percentRange != nil {
if (percentRange.min != nil && size.Percent < *percentRange.min) ||
(percentRange.max != nil && size.Percent > *percentRange.max) {
return SizeSpec{}, fmt.Errorf(
"store size (%s) must be between %f%% and %f%%",
return SizeSpec{}, errors.Newf(
"%s size (%s) must be between %f%% and %f%%",
field,
value,
*percentRange.min,
*percentRange.max,
Expand All @@ -110,16 +112,16 @@ func NewSizeSpec(
var err error
size.InBytes, err = humanizeutil.ParseBytes(value)
if err != nil {
return SizeSpec{}, fmt.Errorf("could not parse store size (%s) %s", value, err)
return SizeSpec{}, errors.Newf("could not parse %s size (%s) %s", field, value, err)
}
if bytesRange != nil {
if bytesRange.min != nil && size.InBytes < *bytesRange.min {
return SizeSpec{}, fmt.Errorf("store size (%s) must be larger than %s", value,
humanizeutil.IBytes(*bytesRange.min))
return SizeSpec{}, errors.Newf("%s size (%s) must be larger than %s",
field, value, humanizeutil.IBytes(*bytesRange.min))
}
if bytesRange.max != nil && size.InBytes > *bytesRange.max {
return SizeSpec{}, fmt.Errorf("store size (%s) must be smaller than %s", value,
humanizeutil.IBytes(*bytesRange.max))
return SizeSpec{}, errors.Newf("%s size (%s) must be smaller than %s",
field, value, humanizeutil.IBytes(*bytesRange.max))
}
}
}
Expand Down Expand Up @@ -150,7 +152,7 @@ var _ pflag.Value = &SizeSpec{}
// Set adds a new value to the StoreSpecValue. It is the important part of
// pflag's value interface.
func (ss *SizeSpec) Set(value string) error {
spec, err := NewSizeSpec(value, nil, nil)
spec, err := NewSizeSpec("specified", value, nil, nil)
if err != nil {
return err
}
Expand All @@ -162,10 +164,11 @@ func (ss *SizeSpec) Set(value string) error {
// StoreSpec contains the details that can be specified in the cli pertaining
// to the --store flag.
type StoreSpec struct {
Path string
Size SizeSpec
InMemory bool
Attributes roachpb.Attributes
Path string
Size SizeSpec
BallastSize *SizeSpec
InMemory bool
Attributes roachpb.Attributes
// StickyInMemoryEngineID is a unique identifier associated with a given
// store which will remain in memory even after the default Engine close
// until it has been explicitly cleaned up by CleanupStickyInMemEngine[s]
Expand All @@ -190,6 +193,7 @@ type StoreSpec struct {

// String returns a fully parsable version of the store spec.
func (ss StoreSpec) String() string {
// TODO(jackson): Implement redact.SafeFormatter
var buffer bytes.Buffer
if len(ss.Path) != 0 {
fmt.Fprintf(&buffer, "path=%s,", ss.Path)
Expand All @@ -203,6 +207,14 @@ func (ss StoreSpec) String() string {
if ss.Size.Percent > 0 {
fmt.Fprintf(&buffer, "size=%s%%,", humanize.Ftoa(ss.Size.Percent))
}
if ss.BallastSize != nil {
if ss.BallastSize.InBytes > 0 {
fmt.Fprintf(&buffer, "ballast-size=%s,", humanizeutil.IBytes(ss.BallastSize.InBytes))
}
if ss.BallastSize.Percent > 0 {
fmt.Fprintf(&buffer, "ballast-size=%s%%,", humanize.Ftoa(ss.BallastSize.Percent))
}
}
if len(ss.Attributes.Attrs) > 0 {
fmt.Fprint(&buffer, "attrs=")
for i, attr := range ss.Attributes.Attrs {
Expand Down Expand Up @@ -308,13 +320,28 @@ func NewStoreSpec(value string) (StoreSpec, error) {
var minPercent float64 = 1
var maxPercent float64 = 100
ss.Size, err = NewSizeSpec(
"store",
value,
&intInterval{min: &minBytesAllowed},
&floatInterval{min: &minPercent, max: &maxPercent},
)
if err != nil {
return StoreSpec{}, err
}
case "ballast-size":
var minBytesAllowed int64
var minPercent float64 = 0
var maxPercent float64 = 50
ballastSize, err := NewSizeSpec(
"ballast",
value,
&intInterval{min: &minBytesAllowed},
&floatInterval{min: &minPercent, max: &maxPercent},
)
if err != nil {
return StoreSpec{}, err
}
ss.BallastSize = &ballastSize
case "attrs":
// Check to make sure there are no duplicate attributes.
attrMap := make(map[string]struct{})
Expand Down Expand Up @@ -384,6 +411,9 @@ func NewStoreSpec(value string) (StoreSpec, error) {
if ss.Size.Percent == 0 && ss.Size.InBytes == 0 {
return StoreSpec{}, fmt.Errorf("size must be specified for an in memory store")
}
if ss.BallastSize != nil {
return StoreSpec{}, fmt.Errorf("ballast-size specified for in memory store")
}
} else if ss.Path == "" {
return StoreSpec{}, fmt.Errorf("no path specified")
}
Expand Down Expand Up @@ -417,6 +447,14 @@ func (ssl StoreSpecList) String() string {
// root directory. It must not be changed without a proper migration.
const AuxiliaryDir = "auxiliary"

// EmergencyBallastFile returns the path (relative to a data directory) used
// for an emergency ballast file. The returned path must be stable across
// releases (eg, we cannot change these constants), otherwise we may duplicate
// ballasts.
func EmergencyBallastFile(pathJoin func(...string) string, dataDir string) string {
return pathJoin(dataDir, AuxiliaryDir, "EMERGENCY_BALLAST")
}

// PreventedStartupFile is the filename (relative to 'dir') used for files that
// can block server startup.
func PreventedStartupFile(dir string) string {
Expand Down
7 changes: 7 additions & 0 deletions pkg/base/store_spec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,13 @@ target_file_size=2097152`
{"size=20GiB,path=/mnt/hda1,size=20GiB", "size field was used twice in store definition", StoreSpec{}},
{"size=123TB", "no path specified", StoreSpec{}},

// ballast size
{"path=/mnt/hda1,ballast-size=671088640", "", StoreSpec{Path: "/mnt/hda1", BallastSize: &SizeSpec{InBytes: 671088640}}},
{"path=/mnt/hda1,ballast-size=20GB", "", StoreSpec{Path: "/mnt/hda1", BallastSize: &SizeSpec{InBytes: 20000000000}}},
{"path=/mnt/hda1,ballast-size=1%", "", StoreSpec{Path: "/mnt/hda1", BallastSize: &SizeSpec{Percent: 1}}},
{"path=/mnt/hda1,ballast-size=100.000%", "ballast size (100.000%) must be between 0.000000% and 50.000000%", StoreSpec{}},
{"ballast-size=20GiB,path=/mnt/hda1,ballast-size=20GiB", "ballast-size field was used twice in store definition", StoreSpec{}},

// type
{"type=mem,size=20GiB", "", StoreSpec{Size: SizeSpec{InBytes: 21474836480}, InMemory: true}},
{"size=20GiB,type=mem", "", StoreSpec{Size: SizeSpec{InBytes: 21474836480}, InMemory: true}},
Expand Down
4 changes: 4 additions & 0 deletions pkg/cli/exit/codes.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,10 @@ func TimeoutAfterFatalError() Code { return Code{8} }
// during a logging operation to a network collector.
func LoggingNetCollectorUnavailable() Code { return Code{9} }

// DiskFull (10) indicates an emergency shutdown in response to a
// store's full disk.
func DiskFull() Code { return Code{10} }

// Codes that are specific to client commands follow. It's possible
// for codes to be reused across separate client or server commands.
// Command-specific exit codes should be allocated down from 125.
Expand Down
52 changes: 52 additions & 0 deletions pkg/cli/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ import (
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
"github.com/cockroachdb/cockroach/pkg/util/tracing"
"github.com/cockroachdb/errors"
"github.com/cockroachdb/errors/oserror"
"github.com/cockroachdb/pebble/vfs"
"github.com/cockroachdb/redact"
"github.com/spf13/cobra"
"google.golang.org/grpc"
Expand Down Expand Up @@ -347,6 +349,16 @@ func runStart(cmd *cobra.Command, args []string, startSingleNode bool) (returnEr
}()
}

// Check for stores with full disks and exit with an informative exit
// code. This needs to happen early during start, before we perform any
// writes to the filesystem including log rotation. We need to guarantee
// that the process continues to exit with the Disk Full exit code. A
// flapping exit code can affect alerting, including the alerting
// performed within CockroachCloud.
if err := exitIfDiskFull(serverCfg.Stores.Specs); err != nil {
return err
}

// Set up a cancellable context for the entire start command.
// The context will be canceled at the end.
ctx, cancel := context.WithCancel(context.Background())
Expand Down Expand Up @@ -1015,6 +1027,46 @@ func maybeWarnMemorySizes(ctx context.Context) {
}
}

func exitIfDiskFull(specs []base.StoreSpec) error {
var cause error
var ballastPaths []string
var ballastMissing bool
for _, spec := range specs {
isDiskFull, err := storage.IsDiskFull(vfs.Default, spec)
if err != nil {
return err
}
if !isDiskFull {
continue
}
path := base.EmergencyBallastFile(vfs.Default.PathJoin, spec.Path)
ballastPaths = append(ballastPaths, path)
if _, err := vfs.Default.Stat(path); oserror.IsNotExist(err) {
ballastMissing = true
}
cause = errors.CombineErrors(cause, errors.Newf(`store %s: out of disk space`, spec.Path))
}
if cause == nil {
return nil
}

// TODO(jackson): Link to documentation surrounding the ballast.

err := clierror.NewError(cause, exit.DiskFull())
if ballastMissing {
return errors.WithHint(err, `At least one ballast file is missing.
You may need to replace this node because there is
insufficient disk space to start.`)
}

ballastPathsStr := strings.Join(ballastPaths, "\n")
err = errors.WithHintf(err, `Deleting or truncating the ballast file(s) at
%s
may reclaim enough space to start. Proceed with caution. Complete
disk space exhaustion may result in node loss.`, ballastPathsStr)
return err
}

// setupAndInitializeLoggingAndProfiling does what it says on the label.
// Prior to this however it determines suitable defaults for the
// logging output directory and the verbosity level of stderr logging.
Expand Down
7 changes: 7 additions & 0 deletions pkg/cli/start_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,13 @@ func TestStartArgChecking(t *testing.T) {
{[]string{`--store=size=-1231MB`}, `store size \(-1231MB\) must be larger than`},
{[]string{`--store=size=1231B`}, `store size \(1231B\) must be larger than`},
{[]string{`--store=size=1231BLA`}, `unhandled size name: bla`},
{[]string{`--store=ballast-size=60.0`}, `ballast size \(60.0\) must be between 0.000000% and 50.000000%`},
{[]string{`--store=ballast-size=1231BLA`}, `unhandled size name: bla`},
{[]string{`--store=ballast-size=0.5%,path=.`}, ``},
{[]string{`--store=ballast-size=.5,path=.`}, ``},
{[]string{`--store=ballast-size=50.%,path=.`}, ``},
{[]string{`--store=ballast-size=50%,path=.`}, ``},
{[]string{`--store=ballast-size=2GiB,path=.`}, ``},
{[]string{`--store=attrs=bli:bli`}, `duplicate attribute`},
{[]string{`--store=type=bli`}, `bli is not a valid store type`},
{[]string{`--store=bla=bli`}, `bla is not a valid store field`},
Expand Down
12 changes: 8 additions & 4 deletions pkg/server/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -529,11 +529,14 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) {
engines = append(engines, e)
}
} else {
if err := vfs.Default.MkdirAll(spec.Path, 0755); err != nil {
return Engines{}, errors.Wrap(err, "creating store directory")
}
du, err := vfs.Default.GetDiskUsage(spec.Path)
if err != nil {
return Engines{}, errors.Wrap(err, "retrieving disk usage")
}
if spec.Size.Percent > 0 {
du, err := vfs.Default.GetDiskUsage(spec.Path)
if err != nil {
return Engines{}, err
}
sizeInBytes = int64(float64(du.TotalBytes) * spec.Size.Percent / 100)
}
if sizeInBytes != 0 && !skipSizeCheck && sizeInBytes < base.MinimumStoreSize {
Expand All @@ -548,6 +551,7 @@ func (cfg *Config) CreateEngines(ctx context.Context) (Engines, error) {
Attrs: spec.Attributes,
Dir: spec.Path,
MaxSize: sizeInBytes,
BallastSize: storage.BallastSizeBytes(spec, du),
Settings: cfg.Settings,
UseFileRegistry: spec.UseFileRegistry,
DisableSeparatedIntents: disableSeparatedIntents,
Expand Down
5 changes: 5 additions & 0 deletions pkg/storage/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go_library(
srcs = [
"array_32bit.go",
"array_64bit.go",
"ballast.go",
"batch.go",
"disk_map.go",
"doc.go",
Expand Down Expand Up @@ -41,6 +42,7 @@ go_library(
visibility = ["//visibility:public"],
deps = [
"//pkg/base",
"//pkg/cli/exit",
"//pkg/clusterversion",
"//pkg/keys",
"//pkg/kv/kvserver/concurrency/lock",
Expand All @@ -63,6 +65,7 @@ go_library(
"//pkg/util/protoutil",
"//pkg/util/stop",
"//pkg/util/syncutil",
"//pkg/util/sysutil",
"//pkg/util/timeutil",
"//pkg/util/tracing",
"//pkg/util/uuid",
Expand All @@ -84,6 +87,7 @@ go_test(
name = "storage_test",
size = "medium",
srcs = [
"ballast_test.go",
"batch_test.go",
"bench_pebble_test.go",
"bench_test.go",
Expand Down Expand Up @@ -137,6 +141,7 @@ go_test(
"//pkg/util/randutil",
"//pkg/util/shuffle",
"//pkg/util/stop",
"//pkg/util/sysutil",
"//pkg/util/timeutil",
"//pkg/util/uint128",
"//pkg/util/uuid",
Expand Down
Loading