Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Show usage per database. #816

Open
wants to merge 11 commits into
base: development
Choose a base branch
from
Open

Show usage per database. #816

Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
c5638c9
Bugfix to bring back ability to show usage per database relationship …
matthiaz Jun 27, 2019
13b4b87
Self review, removing debug info.
matthiaz Jun 27, 2019
c76cb26
remove the useless 100%
matthiaz Jun 27, 2019
6c0d22c
We shouldn't use colours in the database percentages. There are no wa…
matthiaz Jun 27, 2019
0e6defd
A couple more small tweaks to make it just right.
matthiaz Jun 28, 2019
aec9e7d
Merge branch 'development' of https://github.com/platformsh/platforms…
matthiaz Aug 21, 2019
e18e27f
Merge branch 'platformsh-development' into development
matthiaz Aug 21, 2019
ed13f95
I removed too much during merge
matthiaz Aug 21, 2019
834a068
Merge branch 'development' of https://github.com/platformsh/platforms…
matthiaz Aug 29, 2019
19596de
Merge pull request #4 from platformsh/development
matthiaz Sep 18, 2019
13c169c
Moving the disk-distribution of all relations under a flag disk-distr…
matthiaz Sep 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 68 additions & 18 deletions src/Command/Db/DbSizeCommand.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ class DbSizeCommand extends CommandBase
const YELLOW_WARNING_THRESHOLD = 80;//percentage
const BYTE_TO_MEGABYTE = 1048576;
const WASTED_SPACE_WARNING_THRESHOLD = 200;//percentage
const IB_LOG_FILE_SIZE = 104857600; //2*50MB
const IB_TMP_FILE_SIZE = 12582912; //12MB
const MIN_NR_OF_SYS_TABLES = 82;

/**
* {@inheritDoc}
Expand All @@ -29,6 +32,8 @@ protected function configure() {
->setDescription('Estimate the disk usage of a database')
->addOption('bytes', 'B', InputOption::VALUE_NONE, 'Show sizes in bytes.')
->addOption('cleanup', 'C', InputOption::VALUE_NONE, 'Check if tables can be cleaned up and show me recommendations (InnoDb only).')
->addOption('disk-distribution', 'D', InputOption::VALUE_NONE, 'Show all relations and their usage distribution (InnoDb only).')

->setHelp(
"This is an estimate of the database disk usage. The real size on disk is usually a bit higher because of overhead."
);
Expand All @@ -55,6 +60,7 @@ protected function execute(InputInterface $input, OutputInterface $output) {
$this->stdErr->writeln('No database selected.');
return 1;
}

if (!isset($database['service'])) {
$this->stdErr->writeln('Unable to find database service information.');
return 1;
Expand All @@ -71,7 +77,7 @@ protected function execute(InputInterface $input, OutputInterface $output) {
$this->debug('Calculating estimated usage...');
$allocatedDisk = $service->disk * self::BYTE_TO_MEGABYTE;
$estimatedUsage = $this->getEstimatedUsage($host, $database);
$percentageUsed = round($estimatedUsage * 100 / $allocatedDisk);
$percentageUsed = round($estimatedUsage['__TOTAL__'] * 100 / $allocatedDisk);

/** @var \Platformsh\Cli\Service\Table $table */
$table = $this->getService('table');
Expand All @@ -81,16 +87,39 @@ protected function execute(InputInterface $input, OutputInterface $output) {
$columns = ['max' => 'Allocated disk', 'used' => 'Estimated usage', 'percent_used' => '% used'];
$values = [
'max' => $showInBytes ? $allocatedDisk : Helper::formatMemory($allocatedDisk),
'used' => $showInBytes ? $estimatedUsage : Helper::formatMemory($estimatedUsage),
'used' => $showInBytes ? $estimatedUsage['__TOTAL__'] : Helper::formatMemory($estimatedUsage['__TOTAL__']),
'percent_used' => $this->formatPercentage($percentageUsed, $machineReadable),
];

$this->stdErr->writeln('');
$table->render([$values], $columns);

$this->showWarnings($percentageUsed);


if($input->getOption('disk-distribution') && count($relationships->getRelationships($host)) > 1) {
$db_table = $this->getService('table');
$db_columns = ['db' => 'Database', 'used' => 'Estimated usage', 'percent' => 'Percentage'];
$db_values = [];

foreach($estimatedUsage as $db=>$size) {
if($db=='__TOTAL__') {
$db_values[] = new \Symfony\Component\Console\Helper\TableSeparator();
}
$db_values[] = [
'db' => $db==$database['path'] ? sprintf('<options=bold>%s</>',$db) : $db,
'used' => $showInBytes ? $size : Helper::formatMemory($size),
'percent' => $db=='__TOTAL__' ? '' : $this->formatPercentage(round($size * 100 / $estimatedUsage['__TOTAL__']), $machineReadable, false),
];
}

$this->showInaccessibleSchemas($service, $database);
if(count($db_values)) {
$db_table->render($db_values, $db_columns);
} else {
$this->showInaccessibleSchemas($service, $database);//we don't need to do this for InnoDB mysql. We can access the data we need.
}
}



if ($database['scheme'] !== 'pgsql' && $estimatedUsage > 0 && $input->getOption('cleanup')) {
$this->checkInnoDbTablesInNeedOfOptimizing($host, $database);
Expand Down Expand Up @@ -268,8 +297,10 @@ private function mysqlNonInnodbQuery(array $database, $excludeInnoDb = true)
{
$query = 'SELECT'
. ' ('
. 'SUM(data_length+index_length+data_free)'
. ' + (COUNT(*) * 300 * 1024)'
. 'SUM('
. ($excludeInnoDb ? 'IF(ENGINE <> "InnoDB",data_length+index_length+data_free,0)' : 'data_length+index_length+data_free')
. ')'
. ' + ((COUNT(*)+'.self::MIN_NR_OF_SYS_TABLES.') * 450 * 1024) + ' . (self::IB_LOG_FILE_SIZE + self::IB_TMP_FILE_SIZE)
. ')'
. ' AS estimated_actual_disk_usage'
. ' FROM information_schema.tables'
Expand All @@ -287,8 +318,19 @@ private function mysqlNonInnodbQuery(array $database, $excludeInnoDb = true)
*/
private function mysqlInnodbQuery(array $database)
{
$query = 'SELECT SUM(ALLOCATED_SIZE) FROM information_schema.innodb_sys_tablespaces;';

$deployment = $this->api()->getCurrentDeployment($this->getSelectedEnvironment());
$service = $deployment->getService($database['service']);
$databases = array_keys($service->configuration['endpoints'][$database['rel']]['privileges']);

//convert database names to OR statements that the sys_tablespaces can use.
$strDbs = implode(' OR ', array_map(
function($database) {
$database = addslashes($database);
return "`NAME` LIKE \"$database/%\"";
matthiaz marked this conversation as resolved.
Show resolved Hide resolved
},$databases)
);

$query = 'SELECT LEFT(`NAME`, LOCATE("/",`NAME`)-1), SUM(ALLOCATED_SIZE) FROM information_schema.innodb_sys_tablespaces GROUP BY LEFT(`NAME`, LOCATE("/",`NAME`)-1);';
return $this->getMysqlCommand($database, $query);
}

Expand Down Expand Up @@ -325,7 +367,7 @@ private function mysqlTablesInNeedOfOptimizing(array $database) {
* @param HostInterface $host
* @param array $database
*
* @return int Estimated usage in bytes.
* @return array ['__TOTAL__'=>0,...]
*/
private function getEstimatedUsage(HostInterface $host, array $database) {
if ($database['scheme'] === 'pgsql') {
Expand Down Expand Up @@ -357,12 +399,17 @@ private function getPgSqlUsage(HostInterface $host, array $database) {
*/
private function getMySqlUsage(HostInterface $host, array $database) {
$this->debug('Getting MySQL usage...');
$allocatedSizeSupported = $host->runCommand($this->mysqlInnodbAllocatedSizeExists($database));
$innoDbSize = 0;
if ($allocatedSizeSupported) {
$isAllocatedSizeSupported= $host->runCommand($this->mysqlInnodbAllocatedSizeExists($database));
$otherSizes = $host->runCommand($this->mysqlNonInnodbQuery($database, (bool) $isAllocatedSizeSupported));
$innoDbSizes =[];

if ($isAllocatedSizeSupported) {
$this->debug('Checking InnoDB separately for more accurate results...');
try {
$innoDbSize = $host->runCommand($this->mysqlInnodbQuery($database));
foreach(explode(PHP_EOL,$host->runCommand($this->mysqlInnodbQuery($database))) as $row) {
list($dbName, $dbSize) = explode("\t", $row);
$innoDbSizes[$dbName] = $dbSize;
}
} catch (\Symfony\Component\Process\Exception\RuntimeException $e) {
// Some configurations do not have PROCESS privilege(s) and thus have no access to the sys_tablespaces
// table. Ignore MySQL's 1227 Access Denied error, and revert to the legacy calculation.
Expand All @@ -375,9 +422,10 @@ private function getMySqlUsage(HostInterface $host, array $database) {
}
}

$otherSizes = $host->runCommand($this->mysqlNonInnodbQuery($database, (bool) $allocatedSizeSupported));
$otherSizes = $host->runCommand($this->mysqlNonInnodbQuery($database, (bool) $isAllocatedSizeSupported));

return (int) $otherSizes + (int) $innoDbSize;

return $innoDbSizes + ['__OTHER__'=>$otherSizes, '__TOTAL__'=>array_sum($innoDbSizes)+$otherSizes];
}

/**
Expand All @@ -388,12 +436,14 @@ private function getMySqlUsage(HostInterface $host, array $database) {
*
* @return string
*/
private function formatPercentage($percentage, $machineReadable) {
private function formatPercentage($percentage, $machineReadable, $blnUseColour=true) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Irrelevant / unnecessary change

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

$blnUseColour : I use it, because we don't want to show colours when were simply listing the distribution of database usage size.
See: platform db:size -p sudo27zgjpwqa -vvv -e WEBSUPPORT-170-office-renewal --relationship=management -D

if ($machineReadable) {
return round($percentage);
}

if ($percentage > self::RED_WARNING_THRESHOLD) {
if(!$blnUseColour) {
$format = '~ %d%%';
} elseif ($percentage > self::RED_WARNING_THRESHOLD) {
$format = '<options=bold;fg=red>~ %d%%</>';
} elseif ($percentage > self::YELLOW_WARNING_THRESHOLD) {
$format = '<options=bold;fg=yellow>~ %d%%</>';
Expand Down