diff --git a/README.md b/README.md index 562305c9..00a348b1 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ The metrics used in the various VSAN dashboards are now collected every 5 minute Fast. Very fast. That’s what we had in mind when we designed SexiGraf. When you need vSphere metrics, the obvious way is the `PerformanceManager`, but we need something faster so we choosed managed object properties and quickstats like `ResourcePoolQuickStats`. If we have no other choice, we failback to the `PerformanceManager` but we only query the last 15 samples of the `RealTime samplingPeriod` since we pull vSphere metrics every 5 minutes. http://www.sexigraf.fr/vsphere-sexipanels/   -## FreeNAS +## FreeNAS/TrueNAS -Starting from version 9.10, FreeNAS allows users to set a “Remote Graphite Server” target to send all the metrics harvested by Collectd. Guess what would make a nice Graphite target! http://www.sexigraf.fr/freenas-sexipanel/ +Starting from version 9.10, FreeNAS/TrueNAS allows users to set a “Remote Graphite Server” target to send all the metrics harvested by Collectd. Guess what would make a nice Graphite target! http://www.sexigraf.fr/freenas-sexipanel/ ## Windows diff --git a/etc/apache2/sites-available/graphite.conf b/etc/apache2/sites-available/graphite.conf index 50374cbf..ea1d4a14 100644 --- a/etc/apache2/sites-available/graphite.conf +++ b/etc/apache2/sites-available/graphite.conf @@ -118,10 +118,32 @@ ExtendedStatus On ProxyPass /sexiadmin/ ! Alias /sexiadmin/ /var/www/admin/ - Order allow,deny + RewriteEngine On + RewriteCond %{HTTP_REFERER} /sexigraf-web-admin [NC,OR] + RewriteCond %{HTTP_REFERER} /sexiadmin/ [NC] + RewriteRule .* - [L] + RewriteRule .* - [F] + + + ProxyPass /sexihistory/ ! + Alias /sexihistory/ /mnt/wfs/inventory/ + + RewriteEngine On + RewriteCond %{HTTP_REFERER} /sexigraf-web-admin [NC,OR] + RewriteCond %{HTTP_REFERER} /sexiadmin/ [NC] + RewriteRule .* - [L] + RewriteRule .* - [F] + Options Indexes FollowSymLinks + AllowOverride All + Require all granted Allow from all + + Order Allow,Deny + Deny from all + + ProxyPass /favicon.ico ! Alias /favicon.ico /var/www/admin/images/favicon.ico diff --git a/etc/cron.d/xMotionGarbageCollector b/etc/cron.d/xMotionGarbageCollector new file mode 100644 index 00000000..99f91d18 --- /dev/null +++ b/etc/cron.d/xMotionGarbageCollector @@ -0,0 +1 @@ +37 1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/xMotionGarbageCollector.ps1 >/dev/null 2>&1 diff --git a/etc/grafana/provisioning/dashboards/Veeam.yaml b/etc/grafana/provisioning/dashboards/Veeam.yaml new file mode 100644 index 00000000..6e91ecd2 --- /dev/null +++ b/etc/grafana/provisioning/dashboards/Veeam.yaml @@ -0,0 +1,25 @@ +apiVersion: 1 + +providers: + # an unique provider name +- name: 'Veeam' + # org id. will default to orgId 1 if not specified + orgId: 1 + # name of the dashboard folder. Required + folder: 'Veeam Backup & Replication' + # folder UID. will be automatically generated if not specified + # CRC32 of name + folderUid: '30fb6004' + # provider type. Required + type: file + # disable dashboard deletion + disableDeletion: false + # enable dashboard editing + editable: true + # how often Grafana will scan for changed dashboards + updateIntervalSeconds: 10 + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + options: + # path to dashboard files on disk. Required + path: /var/lib/grafana/dashboards/Veeam diff --git a/etc/sexigraf_version b/etc/sexigraf_version index dd9ca0a1..bac3a57d 100644 --- a/etc/sexigraf_version +++ b/etc/sexigraf_version @@ -1 +1 @@ -0.99i "Victory Mine" \ No newline at end of file +0.99j "St. Olga" \ No newline at end of file diff --git a/opt/graphite/conf/storage-aggregation.conf b/opt/graphite/conf/storage-aggregation.conf index c2b18616..affb3ee0 100644 --- a/opt/graphite/conf/storage-aggregation.conf +++ b/opt/graphite/conf/storage-aggregation.conf @@ -41,4 +41,34 @@ aggregationMethod = sum [errorsTx] pattern = .\.errorsTx. xFilesFactor = 0.0 -aggregationMethod = sum \ No newline at end of file +aggregationMethod = sum + +[PathState] +pattern = .\.superstats\.PathState\.* +xFilesFactor = 0.0 +aggregationMethod = last + +[overallStatus] +pattern = .\.quickstats\.overallStatus\.* +xFilesFactor = 0.0 +aggregationMethod = last + +[veeamObjectRestorePoints] +pattern = ^veeam.*\.objectRestorePoints +xFilesFactor = 0.0 +aggregationMethod = sum + +[veeamRestorePointsCount] +pattern = ^veeam.*\.restorePointsCount +xFilesFactor = 0.0 +aggregationMethod = last + +[veeamResult] +pattern = ^veeam.*\.result +xFilesFactor = 0.0 +aggregationMethod = last + +[veeamStatus] +pattern = ^veeam.*\.status +xFilesFactor = 0.0 +aggregationMethod = last diff --git a/opt/graphite/conf/storage-schemas.conf b/opt/graphite/conf/storage-schemas.conf index 98024f30..e113c66a 100644 --- a/opt/graphite/conf/storage-schemas.conf +++ b/opt/graphite/conf/storage-schemas.conf @@ -14,6 +14,8 @@ # Carbon's internal metrics. This entry should match what is specified in # CARBON_METRIC_PREFIX and CARBON_METRIC_INTERVAL settings +# https://m30m.github.io/whisper-calculator/ + [carbon] pattern = ^carbon\. retentions = 60:90d @@ -36,20 +38,24 @@ retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y [vsan_sync] pattern = .*\.vsan\.SyncingVsanObjects\. -retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d +retentions = 5m:24h,10m:48h,30m:96h,60m:7d,240m:30d,720m:90d [vsan_smart] pattern = .*\.vsan\.disk\.smart\. -retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y +retentions = 5m:24h,10m:48h,30m:96h,60m:7d,240m:30d,720m:90d,2880m:1y [vsan_spaceDetail] pattern = .*\.vsan\.spaceDetail\. -retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y +retentions = 5m:24h,10m:48h,30m:96h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y [vsan] pattern = ^vsan\. -retentions = 5m:12h,10m:24h,20m:48h,80m:7d,320m:30d,640m:90d +retentions = 5m:12h,10m:24h,20m:48h,40m:96h,80m:7d,320m:30d,640m:90d + +[veeam] +pattern = ^veeam\. +retentions = 5m:7d,10m:30d,15m:90d,30m:120d,120m:1y [anything_else] pattern = .* -retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y +retentions = 5m:24h,10m:48h,30m:96h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y diff --git a/opt/sexigraf/PullGuestInfo.ps1 b/opt/sexigraf/PullGuestInfo.ps1 index 65de4fa1..cf2bf2af 100644 --- a/opt/sexigraf/PullGuestInfo.ps1 +++ b/opt/sexigraf/PullGuestInfo.ps1 @@ -63,6 +63,10 @@ try { if ($VmwCmdOvfEnvGuest["guestinfo.dns"] -and ([regex]::match($($VmwCmdOvfEnvGuest['guestinfo.dns']), '^([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$')).Success) { $EtcNetworkInterfaces += " dns-nameservers $($VmwCmdOvfEnvGuest['guestinfo.dns'])" + } elseif ($VmwCmdOvfEnvGuest["guestinfo.dns"] -and ([regex]::match($($VmwCmdOvfEnvGuest['guestinfo.dns']), '^([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3}) ([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$')).Success) { + $EtcNetworkInterfaces += " dns-nameservers $($VmwCmdOvfEnvGuest['guestinfo.dns'])" + } elseif ($VmwCmdOvfEnvGuest["guestinfo.dns"] -and ([regex]::match($($VmwCmdOvfEnvGuest['guestinfo.dns']), '^([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3}) ([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3}) ([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$')).Success) { + $EtcNetworkInterfaces += " dns-nameservers $($VmwCmdOvfEnvGuest['guestinfo.dns'])" } if ($VmwCmdOvfEnvGuest["guestinfo.domain"] -and ([regex]::match($($VmwCmdOvfEnvGuest['guestinfo.domain']), '([a-zA-Z]+)')).Success) { diff --git a/opt/sexigraf/VbrConnect.ps1 b/opt/sexigraf/VbrConnect.ps1 new file mode 100644 index 00000000..c5a7baa6 --- /dev/null +++ b/opt/sexigraf/VbrConnect.ps1 @@ -0,0 +1,32 @@ +#!/usr/bin/pwsh -NonInteractive -NoProfile -Command +# +param([Parameter (Mandatory=$true)] [string] $server, [Parameter (Mandatory=$true)] [string] $username, [Parameter (Mandatory=$true)] [string] $password) + +# https://communities.vmware.com/t5/VMware-PowerCLI-Discussions/PowerCLI-on-Debian-Stretch-The-type-initializer-for-VMware/m-p/451739#M10646 +Set-Content -Path Env:HOME -Value '/tmp' + +if (!$(Test-Connection -TargetName $server -TcpPort 9419 -TimeoutSeconds 2)) { + Write-Host "$server is not answering at TCP:9419 check if VeeamBackupRESTSvc is running" + exit 1 +} + +try { + $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} + $VbrBody = @{grant_type = "password";username = $username;password = $password;refresh_token = "";code = "";use_short_term_refresh = ""} + $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody + if ($VbrConnect.access_token) { + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $SessionRefresh = "vbr_" + $server.Replace(".","_") + ".dat" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefresh + Write-Host "Connected to $server" + } else { + Write-Host "Connection to $server failed!" + exit 1 + } +} catch { + # Invoke-RestMethod: Unable to read data from the transport connection: Connection reset by peer. + # https://helpcenter.veeam.com/docs/backup/vbr_rest/tls_certificate.html + Write-Host "$($Error[0])" + exit 1 +} \ No newline at end of file diff --git a/opt/sexigraf/VbrPullStatistics.ps1 b/opt/sexigraf/VbrPullStatistics.ps1 new file mode 100644 index 00000000..1ee46cd3 --- /dev/null +++ b/opt/sexigraf/VbrPullStatistics.ps1 @@ -0,0 +1,477 @@ +#!/usr/bin/pwsh -Command +# +param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) + +$ScriptVersion = "0.9.44" + +$ExecStart = $(Get-Date).ToUniversalTime() +# $stopwatch = [system.diagnostics.stopwatch]::StartNew() + +$ErrorActionPreference = "SilentlyContinue" +$WarningPreference = "SilentlyContinue" +(Get-Process -Id $pid).PriorityClass = 'Idle' +$SexiVbrMutex = New-Object System.Threading.Mutex($false, "SexiVbrMutex") +# https://learn-powershell.net/2014/09/30/using-mutexes-to-write-data-to-the-same-logfile-across-processes-with-powershell/ +# https://github.com/ryan-leap/GreenMeansGoMutexDemo + +function SexiLogger { + param($Text2Log) + Write-Host "$((Get-Date).ToString("o")) $Text2Log" + $null = $SexiVbrMutex.WaitOne(500) + Add-Content -Path "/var/log/sexigraf/VbrPullStatistics.log" -Value "$((Get-Date).ToString("o")) $($Server) $Text2Log" + $SexiVbrMutex.ReleaseMutex() +} + +function AltAndCatchFire { + Param($ExitReason) + SexiLogger "[EROR] $ExitReason" + SexiLogger "[EROR] $($Error[0])" + SexiLogger "[EROR] Exit" + Stop-Transcript + exit +} + +function NameCleaner { + Param($NameToClean) + $NameToClean = $NameToClean -replace "[ .]","_" + [System.Text.NormalizationForm]$NormalizationForm = "FormD" + $NameToClean = $NameToClean.Normalize($NormalizationForm) + $NameToClean = $NameToClean -replace "[^[:ascii:]]","" -replace "[^A-Za-z0-9-_]","_" + return $NameToClean.ToLower() +} + + +try { + Start-Transcript -Path "/var/log/sexigraf/VbrPullStatistics.$($Server).log" -Append -Confirm:$false -Force -UseMinimalHeader + SexiLogger "[INFO] VbrPullStatistics v$ScriptVersion" +} catch { + SexiLogger "[EROR] VbrPullStatistics logging failure" + SexiLogger "[EROR] Exit" + exit +} + +try { + SexiLogger "[INFO] Importing Graphite PowerShell module ..." + Import-Module -Name /usr/local/share/powershell/Modules/Graphite-PowerShell-Functions/Graphite-Powershell.psm1 -Global -Force -SkipEditionCheck +} catch { + AltAndCatchFire "Powershell modules import failure" +} + +try { + SexiLogger "[INFO] Looking for another VbrPullStatistics for $Server ..." + $DupVbrPullStatisticsProcess = Get-PSHostProcessInfo|%{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}|?{$_ -match "VbrPullStatistics" -and $_ -match "$Server"} + # https://github.com/PowerShell/PowerShell/issues/13944 + if (($DupVbrPullStatisticsProcess|Measure-Object).Count -gt 1) { + $DupVbrPullStatisticsProcessId = (Get-PSHostProcessInfo|?{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '|?{$_ -match "$Server"}}).ProcessId[0] + $DupVbrPullStatisticsProcessTime = [INT32](ps -p $DupVbrPullStatisticsProcessId -o etimes).split()[-1] + if ($DupVbrPullStatisticsProcessTime -gt 300) { + SexiLogger "[WARN] VbrPullStatistics for $Server is already running for more than 5 minutes!" + SexiLogger "[WARN] Killing stunned VbrPullStatistics for $Server" + Stop-Process -Id $DupVbrPullStatisticsProcessId -Force + } else { + AltAndCatchFire "VbrPullStatistics for $Server is already running!" + } + } +} catch { + AltAndCatchFire "VbrPullStatistics process lookup failure" +} + +if ($SessionFile) { + try { + $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" + if ([DateTime]$(Get-Content -Path /tmp/$SessionSecretExpiration) -gt $ExecStart.AddMinutes(5)) { + $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop + SexiLogger "[INFO] SessionToken found in SessionFile, attempting connection to $Server ..." + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders + if ($($VbrJobsStates.data)) { + SexiLogger "[INFO] Connected to VBR REST API Server $Server" + } else { + SexiLogger "[WARN] Connection failure or no job state" + } + } else { + SexiLogger "[WARN] Token has expired or is about to ..." + } + } catch { + SexiLogger "[WARN] SessionToken or SessionSecretExpiration not found, invalid or connection failure" + } + + if (!$($VbrJobsStates.data)) { + SexiLogger "[WARN] Attempting token refresh ..." + try { + $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" + $SessionRefresh = Get-Content -Path /tmp/$SessionRefreshPath -ErrorAction Stop + $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} + $VbrBody = @{grant_type = "refresh_token";username = "";password = "";refresh_token = $SessionRefresh;code = "";use_short_term_refresh = ""} + $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody + if ($VbrConnect.access_token) { + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefreshPath + $VbrConnect.".expires".ToUniversalTime().tostring() | Out-File -FilePath /tmp/$SessionSecretExpiration + $SessionToken = $VbrConnect.access_token + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders + if (!$($VbrJobsStates.data)) { + SexiLogger "[WARN] Token refresh failed!" + SexiLogger "[WARN] Known issue on v11!" + # https://forums.veeam.com/restful-api-f30/how-to-handle-the-refresh-token-t74916.html + } + } else { + SexiLogger "[WARN] Token refresh failed!" + } + } catch { + SexiLogger "[WARN] Token refresh issue!" + } + } + + if (!$($VbrJobsStates.data)) { + SexiLogger "[WARN] Attempting explicit connection ..." + try { + $createstorexml = New-Object -TypeName XML + $createstorexml.Load($credstore) + $XPath = '//passwordEntry[server="' + $Server + '"]' + if ($(Select-XML -Xml $createstorexml -XPath $XPath)){ + $item = Select-XML -Xml $createstorexml -XPath $XPath + $CredStoreLogin = $item.Node.username + $CredStorePassword = [System.Text.Encoding]::Unicode.GetString([System.Convert]::FromBase64String($item.Node.password)) + } else { + AltAndCatchFire "No $Server entry in CredStore" + } + $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} + $VbrBody = @{grant_type = "password";username = $CredStoreLogin;password = $CredStorePassword;refresh_token = "";code = "";use_short_term_refresh = ""} + $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody + if ($VbrConnect.access_token) { + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders + if ($($VbrJobsStates.data)) { + SexiLogger "[INFO] Connected to VBR REST API Server $Server" + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" + $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefreshPath + $VbrConnect.".expires".ToUniversalTime().tostring() | Out-File -FilePath /tmp/$SessionSecretExpiration + $SessionToken = $VbrConnect.access_token + } else { + AltAndCatchFire "Jobs States check failed, no job state or check the user permissions!" + } + } else { + AltAndCatchFire "Explicit connection failed, check the stored credentials!" + } + } catch { + AltAndCatchFire "Explicit connection failed, check the stored credentials!" + } + } +} else { + AltAndCatchFire "No SessionFile somehow ..." +} + +if ($($VbrJobsStates.data)) { + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} + $VbrDataTable = @{} + $vbrserver_name = NameCleaner $Server + SexiLogger "[INFO] Start processing VBR Server $Server ..." + + $VbrJobsStatesTable = @{} + foreach ($VbrJobState in $VbrJobsStates.data) { + try { + $VbrJobsStatesTable.add($VbrJobState.id,$VbrJobState) + } catch {} + $job_name = NameCleaner $VbrJobState.name + if ($VbrJobState.status -eq "running") { + $VbrJobStateStatus = 0 + } elseif ($VbrJobState.status -eq "inactive") { + $VbrJobStateStatus = 1 + } elseif ($VbrJobState.status -eq "disabled") { + $VbrJobStateStatus = 2 + } else { + $VbrJobStateStatus = 3 + } + $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.status"] = $VbrJobStateStatus + } + + try { + SexiLogger "[INFO] VBR repositories states collect ..." + $VbrRepositoriesStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupInfrastructure/repositories/states") -Headers $VbrAuthHeaders + } catch { + SexiLogger "[EROR] repositories states collect failure" + SexiLogger "[EROR] $($Error[0])" + } + + if ($VbrRepositoriesStates.data) { + SexiLogger "[INFO] VBR repositories processing ..." + $VbrRepositoryTable = @{} + foreach ($VbrRepository in $VbrRepositoriesStates.data) { + $VbrRepositoryName = NameCleaner $VbrRepository.name + + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.capacityGB"] = $VbrRepository.capacityGB + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.freeGB"] = $VbrRepository.freeGB + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.usedSpaceGB"] = $VbrRepository.usedSpaceGB + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.RealUsedPct"] = $($VbrRepository.usedSpaceGB * 100 / $($VbrRepository.usedSpaceGB + $VbrRepository.freeGB )) + + try { + $VbrRepositoryTable.add($VbrRepository.id,$VbrRepository) + } catch {} + } + } else { + SexiLogger "[WARN] No repositories ?!" + } + + try { + SexiLogger "[INFO] VBR ScaleOutRepositories collect ..." + $VbrScaleOutRepositories = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupInfrastructure/scaleOutRepositories") -Headers $VbrAuthHeaders + } catch { + SexiLogger "[EROR] ScaleOutRepositories collect failure" + SexiLogger "[EROR] $($Error[0])" + } + + if ($VbrScaleOutRepositories.data) { + SexiLogger "[INFO] SOBR performanceTier processing ..." + foreach ($VbrScaleOutRepository in $VbrScaleOutRepositories.data) { + try { + $VbrScaleOutRepositoryName = NameCleaner $VbrScaleOutRepository.name + + $VbrSobrNotNormal = $VbrScaleOutRepository.performanceTier.performanceExtents|?{$_.status -ne "Normal"} + + if ($VbrSobrNotNormal.id) { + $VbrSobrNotNormalRepos = $VbrRepositoryTable[$VbrSobrNotNormal.id] + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.capacityGB"] += $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property capacityGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] += $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property usedSpaceGB).Sum + } + + $VbrSobrNormal = $VbrScaleOutRepository.performanceTier.performanceExtents|?{$_.status -eq "Normal"} + if ($VbrSobrNormal.id) { + $VbrSobrNormalRepos = $VbrRepositoryTable[$VbrSobrNormal.id] + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.capacityGB"] += $($VbrSobrNormalRepos|Measure-Object -Sum -Property capacityGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] += $($VbrSobrNormalRepos|Measure-Object -Sum -Property usedSpaceGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"] = $($VbrSobrNormalRepos|Measure-Object -Sum -Property freeGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.RealUsedPct"] = $($VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] * 100 / $($VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"])) + } else { + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"] = 0 + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.RealUsedPct"] = 100 + } + } catch { + SexiLogger "[EROR] Issue processing SOBR $($VbrScaleOutRepository.name)" + SexiLogger "[EROR] $($Error[0])" + } + } + } else { + SexiLogger "[INFO] No SOBR" + } + + try { + SexiLogger "[INFO] VBR 5min old objectRestorePoints collect ..." + # $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + $VbrObjectRestorePoints5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/objectRestorePoints?platformNameFilter=VmWare&limit=999&createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + } catch { + SexiLogger "[EROR] objectRestorePoints collect failure" + SexiLogger "[EROR] $($Error[0])" + } + + # $VbrVmwareServers = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/inventory/vmware/hosts") -Headers $VbrAuthHeaders + + + if ($VbrObjectRestorePoints5.data) { + # SexiLogger "[INFO] VBR backupObjects collect ..." + # $VbrBackupObjects5 = @{} + # foreach ($VbrObjectRestorePoint5 in $VbrObjectRestorePoints5.data) { + # try { + # $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&nameFilter=" + $($VbrObjectRestorePoint5.name)) -Headers $VbrAuthHeaders + # if ($VbrBackupObjects.data[0].type -eq "VM" -and !$VbrBackupObjects5[$VbrBackupObjects.data[0].name]) { + # $VbrBackupObjects5.Add($VbrBackupObjects.data[0].name,$VbrBackupObjects[0].data) + # } + # } catch { + # SexiLogger "[EROR] backupObjects collect failure" + # SexiLogger "[EROR] $($Error[0])" + # } + # } # Too Slow !!! + + try { + SexiLogger "[INFO] VBR backupObjects collect ..." + $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&limit=9999") -Headers $VbrAuthHeaders # TODO limit=9999+ ? + if ($VbrBackupObjects.data) { + SexiLogger "[INFO] Building VBR backupObjects table ..." + $VbrBackupObjectsTable = @{} + foreach ($VbrBackupObject in $VbrBackupObjects.data) { + try { + $VbrBackupObjectsTable.add($VbrBackupObject.name,$VbrBackupObject) + } catch {} + } + } + } catch { + SexiLogger "[EROR] backupObjects collect failure" + SexiLogger "[EROR] $($Error[0])" + } + + $VbrObjectRestorePoints5SessionsId = $VbrObjectRestorePoints5.data.backupId|Select-Object -Unique + + SexiLogger "[INFO] VBR Sessions collect ..." + $VbrSessions5 = @{} + foreach ($VbrObjectRestorePoints5SessionId in $VbrObjectRestorePoints5SessionsId) { + try { + $VbrObjectRestorePoints5Session = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backups/" + $VbrObjectRestorePoints5SessionId) -Headers $VbrAuthHeaders + if ($VbrObjectRestorePoints5Session|?{$_.jobId -ne "00000000-0000-0000-0000-000000000000"}) { + $VbrSessions5.Add($VbrObjectRestorePoints5SessionId,$VbrObjectRestorePoints5Session) + } + } catch { + SexiLogger "[EROR] backupObjects collect failure" + SexiLogger "[EROR] $($Error[0])" + } + } + + if (Test-Path /mnt/wfs/inventory/ViVmInventory.csv) { + SexiLogger "[INFO] Importing VM inventory ..." + try { + $ViVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop) + } catch { + SexiLogger "[EROR] ViVmInventory import issue" + SexiLogger "[EROR] $($Error[0])" + } + + if ($ViVmInventory) { + $ViVmInventoryTable = @{} + foreach ($ViVm in $ViVmInventory) { + try { + $ViVmInventoryTable.Add($ViVm.vm,$ViVm) + } catch {} + } + + if (Test-Path /mnt/wfs/inventory/VbrInv_$($Server).csv) { + SexiLogger "[INFO] Importing $($Server) VBR inventory ..." + try { + $VbrVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/VbrInv_$($Server).csv -ErrorAction Stop) + } catch { + SexiLogger "[EROR] VbrVmInventory import issue" + SexiLogger "[EROR] $($Error[0])" + } + if ($ViVmInventory) { + $VbrVmInventoryTable = @{} + foreach ($VbrVm in $VbrVmInventory) { + try { + if ($ViVmInventoryTable[$VbrVm.vm]) { + $VbrVmInventoryTable.Add($VbrVm.vm,$VbrVm) + } + } catch {} + } + } + } else { + $VbrVmInventoryTable = @{} + } + + SexiLogger "[INFO] Building $($Server) VBR DataTable and inventory ..." + foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { + if ($VbrJobsStatesTable[$VbrSessions5[$VbrObjectRestorePoint.backupId].jobId]) { + $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name + $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ + $vm_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name + if ($ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name]) { + $vcenter_name = NameCleaner $ViVmInventoryTable[$vm_name].vCenter + if ($ViVmInventoryTable[$vm_name].Cluster) { + $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].Cluster + } else { + $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].ESX + } + $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.objectRestorePoints"] ++ + $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount + + $VbrObjectInventoryInfo = "" | Select-Object VbrServer, JobName, vCenter, Cluster, VM, RestorePointsCount, LastRestorePoint + $VbrObjectInventoryInfo.VbrServer = $Server + $VbrObjectInventoryInfo.JobName = $VbrSessions5[$VbrObjectRestorePoint.backupId].name + $VbrObjectInventoryInfo.vCenter = $ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name].vCenter + $VbrObjectInventoryInfo.Cluster = $ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name].Cluster + $VbrObjectInventoryInfo.VM = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name + $VbrObjectInventoryInfo.RestorePointsCount = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount + $VbrObjectInventoryInfo.LastRestorePoint = $VbrObjectRestorePoint.creationTime + if ($VbrVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name]) { + $VbrVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name] = $VbrObjectInventoryInfo + } else { + $VbrVmInventoryTable.add($VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name,$VbrObjectInventoryInfo) + } + } + } + } + + if ($VbrVmInventoryTable.Values) { + SexiLogger "[INFO] Exporting $($Server) VBR inventory ..." + try { + $VbrVmInventoryTable.Values|Sort-Object VM|Export-Csv /mnt/wfs/inventory/VbrInv_$($Server).csv -ErrorAction Stop -Force + } catch { + SexiLogger "[EROR] VbrVmInventory import issue" + SexiLogger "[EROR] $($Error[0])" + } + + $VbrVmInventories = Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv" + + if ($VbrVmInventories) { + SexiLogger "[INFO] Rotating VbrVmInventory.*.csv files ..." + $ExtraCsvFiles = Compare-Object $VbrVmInventories $($VbrVmInventories|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} + If ($ExtraCsvFiles) { + try { + Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false + } catch { + AltAndCatchFire "Cannot remove extra csv files" + } + } + } + + SexiLogger "[INFO] Import/Export VBR inventories ..." + try { + $VbrVmInventoryCsv = Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Sort-Object LastRestorePoint -Descending + $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force + if ($ExecStart.DayOfWeek -match "Monday" -and !$($VbrVmInventories|?{$_.LastWriteTime -gt $ExecStart.AddDays(-1)})) { + $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -ErrorAction Stop -Force + } + } catch { + SexiLogger "[EROR] Import/Export VBR inventories issue" + SexiLogger "[EROR] $($Error[0])" + } + } else { + SexiLogger "[EROR] No VbrVmInventoryTable values ?!" + } + } else { + SexiLogger "[EROR] ViVmInventory import issue" + } + } else { + SexiLogger "[EROR] No ViVmInventory" + } + } + + try { + SexiLogger "[INFO] VBR ended sessions collect ..." + $VbrEndedSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?endedAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + } catch { + SexiLogger "[EROR] Sessions collect failure" + SexiLogger "[EROR] $($Error[0])" + } + + if ($VbrEndedSessions.data) { + SexiLogger "[INFO] Processing VBR ended sessions ..." + foreach ($VbrEndedSession in $VbrEndedSessions.data) { + $job_name = NameCleaner $VbrEndedSession.name + if ($VbrEndedSession.result.result -eq "Success") { + $VbrEndedSessionResult = 0 + } elseif ($VbrEndedSession.result.result -eq "Warning") { + $VbrEndedSessionResult = 1 + } elseif ($VbrEndedSession.result.result -eq "Failed") { + $VbrEndedSessionResult = 2 + } else { + $VbrEndedSessionResult = 3 + } + $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.result"] = $VbrEndedSessionResult + } + } + + $VbrDataTable["veeam.vbr.$vbrserver_name.exec.duration"] = $($(Get-Date).ToUniversalTime() - $ExecStart).TotalSeconds + SexiLogger "[INFO] Sending veeam data to Graphite for VBR server $Server ..." + Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrDataTable -DateTime $ExecStart + + SexiLogger "[INFO] End of VBR server $server processing ..." + +} else { + SexiLogger "[INFO] No Running Jobs on VBR server $server ..." + SexiLogger "[INFO] Exit" + Stop-Transcript + exit +} \ No newline at end of file diff --git a/opt/sexigraf/VbrVmInventory.csv b/opt/sexigraf/VbrVmInventory.csv new file mode 100644 index 00000000..01ce379d --- /dev/null +++ b/opt/sexigraf/VbrVmInventory.csv @@ -0,0 +1 @@ +"VbrServer","JobName","vCenter","Cluster","VM","RestorePointsCount","LastRestorePoint" \ No newline at end of file diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 6eb33409..2f4bf464 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -1,9 +1,9 @@ -#!/usr/bin/pwsh -Command[EROR] +#!/usr/bin/pwsh -Command # param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.69" +$ScriptVersion = "0.9.90" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -250,9 +250,9 @@ if ($ViServersList.count -gt 0) { $VmCluster = "" } - if ($Vm.Guest.GuestId) { + if ($Vm.Guest.GuestId.Length -gt 0) { $VmGuestId = $Vm.Guest.GuestId.replace('Guest','') - } elseif ($Vm.Config.GuestId) { + } elseif ($Vm.Config.GuestId -gt 0) { $VmGuestId = $Vm.Config.GuestId.replace('Guest','') } else { $VmGuestId = "" @@ -273,12 +273,12 @@ if ($ViServersList.count -gt 0) { $ViVmInfo.Cluster = $VmCluster $ViVmInfo.IP = $VmIpAddress -join " ; " $ViVmInfo.PortGroup = $VmNet -join " ; " - $ViVmInfo.Committed_GB = [math]::round($Vm.Summary.Storage.committed/1GB,1) - $ViVmInfo.Allocated_GB = [math]::round(($Vm.Summary.Storage.Committed + $Vm.Summary.Storage.Uncommitted)/1GB,1) + $ViVmInfo.Committed_GB = [math]::round($Vm.Summary.Storage.committed/1GB,0) + $ViVmInfo.Allocated_GB = [math]::round(($Vm.Summary.Storage.Committed + $Vm.Summary.Storage.Uncommitted)/1GB,0) $ViVmInfo.MAC = ($Vm.Config.Hardware.Device|?{$_.MacAddress}).MacAddress -join " ; " $ViVmInfo.GuestId = $VmGuestId $ViVmInfo.vCPU = $Vm.Config.Hardware.NumCPU - $ViVmInfo.vRAM_GB = [math]::round($Vm.Config.Hardware.MemoryMB/1KB,1) + $ViVmInfo.vRAM_GB = [math]::round($Vm.Config.Hardware.MemoryMB/1KB,0) $ViVmInfo.PowerState = $Vm.Runtime.PowerState $ViVmInfo.vmxPath = $Vm.summary.config.vmPathName $ViVmInfo.Folder = $VmPath @@ -324,7 +324,7 @@ if ($ViServersList.count -gt 0) { $ViEsxInfo.Model = $Esx.Summary.Hardware.Model $ViEsxInfo.SerialNumber = $EsxServiceTag $ViEsxInfo.State = $EsxState - $ViEsxInfo.RAM_GB = [math]::round($Esx.Summary.Hardware.MemorySize/1GB,1) + $ViEsxInfo.RAM_GB = [math]::round($Esx.Summary.Hardware.MemorySize/1GB,0) $ViEsxInfo.CPU = $Esx.Summary.Hardware.CpuModel $ViEsxInfo.Cores = $Esx.Summary.Hardware.NumCpuCores $ViEsxInfo.vmk0Ip = ($Esx.Config.Network.Vnic|?{$_.Device -eq "vmk0"}).Spec.Ip.IpAddress @@ -340,9 +340,9 @@ if ($ViServersList.count -gt 0) { $ViDatastoreInfo.vCenter = $ViServer $ViDatastoreInfo.Datastore = $($Datastore.name) $ViDatastoreInfo.Type = $($Datastore.Summary.Type) - $ViDatastoreInfo.Capacity_GB = $([math]::round($Datastore.Summary.Capacity/1GB,1)) - $ViDatastoreInfo.FreeSpace_GB = $([math]::round($Datastore.Summary.FreeSpace/1GB,1)) - $ViDatastoreInfo."Usage_%" = $([math]::round(($Datastore.Summary.Capacity - $Datastore.Summary.FreeSpace) * 100 / $Datastore.Summary.Capacity,1)) + $ViDatastoreInfo.Capacity_GB = $([math]::round($Datastore.Summary.Capacity/1GB,0)) + $ViDatastoreInfo.FreeSpace_GB = $([math]::round($Datastore.Summary.FreeSpace/1GB,0)) + $ViDatastoreInfo."Usage_%" = $([math]::round(($Datastore.Summary.Capacity - $Datastore.Summary.FreeSpace) * 100 / $Datastore.Summary.Capacity,0)) $ViDatastoreInfo.Url = $($Datastore.Summary.Url) $ViDatastoresInfos += $ViDatastoreInfo @@ -361,32 +361,133 @@ if ($ViServersList.count -gt 0) { } if ($ViVmsInfos) { + + $ViVmInventories = Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv" + + if ($ViVmInventories) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViVmInventory.*.csv files ..." + $ExtraCsvFiles = Compare-Object $ViVmInventories $($ViVmInventories|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} + If ($ExtraCsvFiles) { + try { + Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false + } catch { + AltAndCatchFire "Cannot remove extra csv files" + } + } + } + try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory files ..." - $ViVmsInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViVmInventory.csv -Force + Write-Host "$((Get-Date).ToString("o")) [INFO] Building Vm Inventory CSV ..." + $ViVmsInfosCsv = $ViVmsInfos|ConvertTo-Csv -NoTypeInformation -ErrorAction Stop + # Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for vCenter,Cluster,VM differences with previous inventory file ..." + # if (Test-Path -Path /mnt/wfs/inventory/ViVmInventory.csv) { + # $ViVmsInfosCsvBak = Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop + # if ($ViVmsInfosCsvBak) { + # if (Compare-Object $ViVmsInfosCsvBak $ViVmsInfos -Property vCenter,Cluster,VM) { + # Write-Host "$((Get-Date).ToString("o")) [INFO] Differences detected with previous inventory file ..." + # $VmMigratedScan = $true + # } else { + # $VmMigratedScan = $false + # Write-Host "$((Get-Date).ToString("o")) [INFO] No differences detected with previous inventory file ..." + # } + # } else { + # Write-Host "$((Get-Date).ToString("o")) [EROR] Empty Vm Inventory file" + # $VmMigratedScan = $true + # } + # } else { + # Write-Host "$((Get-Date).ToString("o")) [EROR] No existing Vm Inventory file" + # $VmMigratedScan = $true + # } + Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory file ..." + $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.csv -Force -ErrorAction Stop + if ($ExecStart.DayOfWeek -match "Monday" -and !$($ViVmInventories|?{$_.LastWriteTime -gt $ExecStart.AddDays(-1)})) { + $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop + } } catch { - AltAndCatchFire "VM Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] VM Inventory issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } } if ($ViEsxsInfos) { + + $ViEsxInventories = Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv" + + if ($ViEsxInventories) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViEsxInventory.*.csv files ..." + $ExtraCsvFiles = Compare-Object $ViEsxInventories $($ViEsxInventories|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} + If ($ExtraCsvFiles) { + try { + Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false + } catch { + AltAndCatchFire "Cannot remove extra csv files" + } + } + } + try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Esx Inventory files ..." - $ViEsxsInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force + Write-Host "$((Get-Date).ToString("o")) [INFO] Building ESX Inventory CSV ..." + $ViEsxsInfosCsv = $ViEsxsInfos|ConvertTo-Csv -NoTypeInformation -ErrorAction Stop + # Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for vCenter,Cluster,ESX differences with previous inventory file ..." + # if (Test-Path -Path /mnt/wfs/inventory/ViEsxInventory.csv) { + # $ViEsxsInfosCsvBak = Import-Csv -Path /mnt/wfs/inventory/ViEsxInventory.csv -ErrorAction Stop + # if ($ViEsxsInfosCsvBak) { + # if (Compare-Object $ViEsxsInfosCsvBak $ViEsxsInfos -Property vCenter,Cluster,ESX) { + # Write-Host "$((Get-Date).ToString("o")) [INFO] Differences detected with previous inventory file ..." + # $EsxMigratedScan = $true + # } else { + # $EsxMigratedScan = $false + # Write-Host "$((Get-Date).ToString("o")) [INFO] No differences detected with previous inventory file ..." + # } + # } else { + # Write-Host "$((Get-Date).ToString("o")) [EROR] Empty ESX Inventory file" + # $EsxMigratedScan = $true + # } + # } else { + # Write-Host "$((Get-Date).ToString("o")) [EROR] No existing ESX Inventory file" + # $EsxMigratedScan = $true + # } + Write-Host "$((Get-Date).ToString("o")) [INFO] Writing ESX Inventory file ..." + $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force -ErrorAction Stop + if ($ExecStart.DayOfWeek -match "Monday" -and !$($ViEsxInventories|?{$_.LastWriteTime -gt $ExecStart.AddDays(-1)})) { + $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop + } } catch { - AltAndCatchFire "ESX Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX Inventory issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } } if ($ViDatastoresInfos) { + + $ViDsInventories = Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv" + + if ($ViDsInventories) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViDsInventory.*.csv files ..." + $ExtraCsvFiles = Compare-Object $ViDsInventories $($ViDsInventories|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} + If ($ExtraCsvFiles) { + try { + Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false + } catch { + AltAndCatchFire "Cannot remove extra csv files" + } + } + } + try { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Datastore Inventory files ..." - $ViDatastoresInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViDsInventory.csv -Force + $ViDatastoresInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViDsInventory.csv -Force -ErrorAction Stop + if ($ExecStart.DayOfWeek -match "Monday" -and !$($ViDsInventories|?{$_.LastWriteTime -gt $ExecStart.AddDays(-1)})) { + $ViDatastoresInfos|Out-File -Path /mnt/wfs/inventory/ViDsInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop + } } catch { - AltAndCatchFire "Datastore Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] Datastore Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } } + Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf ViOfflineInventory has left the building ..." + } else { AltAndCatchFire "No VI server to process" } \ No newline at end of file diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 46645c20..2795e715 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -1,8 +1,8 @@ #!/usr/bin/pwsh -Command # -param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) +param([parameter (Mandatory=$true)] [string] $Server, [parameter (Mandatory=$true)] [string] $SessionFile, [parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1006" +$ScriptVersion = "0.9.1035" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -10,18 +10,29 @@ $ExecStart = $(Get-Date).ToUniversalTime() $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" (Get-Process -Id $pid).PriorityClass = 'Idle' +$SexiViMutex = New-Object System.Threading.Mutex($false, "SexiViMutex") +# https://learn-powershell.net/2014/09/30/using-mutexes-to-write-data-to-the-same-logfile-across-processes-with-powershell/ +# https://github.com/ryan-leap/GreenMeansGoMutexDemo + +function SexiLogger { + param($Text2Log) + Write-Host "$((Get-Date).ToString("o")) $Text2Log" + $null = $SexiViMutex.WaitOne(500) + Add-Content -Path "/var/log/sexigraf/ViPullStatistics.log" -Value "$((Get-Date).ToString("o")) $($Server) $Text2Log" + $SexiViMutex.ReleaseMutex() +} function AltAndCatchFire { - Param($ExitReason) - Write-Host "$((Get-Date).ToString("o")) [EROR] $ExitReason" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" - Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + param($ExitReason) + SexiLogger "[EROR] $ExitReason" + SexiLogger "[EROR] $($Error[0])" + SexiLogger "[EROR] Exit" Stop-Transcript exit } function GetRootDc { - Param($child_object) + param($child_object) if ($xfolders_vcenter_parent_h[$child_object.Parent.value]) { $Parent_folder = $child_object.Parent.value while ($xfolders_vcenter_type_h[$xfolders_vcenter_parent_h[$Parent_folder]] -notmatch "^Datacenter$") { @@ -32,7 +43,7 @@ function GetRootDc { } function NameCleaner { - Param($NameToClean) + param($NameToClean) $NameToClean = $NameToClean -replace "[ .]","_" [System.Text.NormalizationForm]$NormalizationForm = "FormD" $NameToClean = $NameToClean.Normalize($NormalizationForm) @@ -171,20 +182,15 @@ function MultiQueryPerf300 { try { Start-Transcript -Path "/var/log/sexigraf/ViPullStatistics.$($Server).log" -Append -Confirm:$false -Force -UseMinimalHeader - Start-Transcript -Path "/var/log/sexigraf/ViPullStatistics.log" -Append -Confirm:$false -Force -UseMinimalHeader - Write-Host "$((Get-Date).ToString("o")) [INFO] ViPullStatistics v$ScriptVersion" - if ( $vSanPull = Test-Path -Path $("/etc/cron.d/vsan_" + $Server.Replace(".","_"))) { - Start-Transcript -Path "/var/log/sexigraf/VsanDisksPullStatistics.$($Server).log" -Append -Confirm:$false -Force -UseMinimalHeader - Start-Transcript -Path "/var/log/sexigraf/VsanDisksPullStatistics.log" -Append -Confirm:$false -Force -UseMinimalHeader - } + SexiLogger "[INFO] ViPullStatistics v$ScriptVersion" } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ViPullStatistics logging failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + SexiLogger "[EROR] ViPullStatistics logging failure" + SexiLogger "[EROR] Exit" exit } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Importing PowerCli and Graphite PowerShell modules ..." + SexiLogger "[INFO] Importing PowerCli and Graphite PowerShell modules ..." Import-Module VMware.VimAutomation.Common, VMware.VimAutomation.Core, VMware.VimAutomation.Sdk, VMware.VimAutomation.Storage $PowerCliConfig = Set-PowerCLIConfiguration -ProxyPolicy NoProxy -DefaultVIServerMode Single -InvalidCertificateAction Ignore -ParticipateInCeip:$false -DisplayDeprecationWarnings:$false -Confirm:$false -Scope Session Import-Module -Name /usr/local/share/powershell/Modules/Graphite-PowerShell-Functions/Graphite-Powershell.psm1 -Global -Force -SkipEditionCheck @@ -193,15 +199,15 @@ try { } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for another ViPullStatistics for $Server ..." + SexiLogger "[INFO] Looking for another ViPullStatistics for $Server ..." $DupViPullStatisticsProcess = Get-PSHostProcessInfo|%{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}|?{$_ -match "ViPullStatistics" -and $_ -match "$Server"} # https://github.com/PowerShell/PowerShell/issues/13944 if (($DupViPullStatisticsProcess|Measure-Object).Count -gt 1) { $DupViPullStatisticsProcessId = (Get-PSHostProcessInfo|?{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '|?{$_ -match "$Server"}}).ProcessId[0] $DupViPullStatisticsProcessTime = [INT32](ps -p $DupViPullStatisticsProcessId -o etimes).split()[-1] if ($DupViPullStatisticsProcessTime -gt 300) { - Write-Host "$((Get-Date).ToString("o")) [WARN] ViPullStatistics for $Server is already running for more than 5 minutes!" - Write-Host "$((Get-Date).ToString("o")) [WARN] Killing stunned ViPullStatistics for $Server" + SexiLogger "[WARN] ViPullStatistics for $Server is already running for more than 5 minutes!" + SexiLogger "[WARN] Killing stunned ViPullStatistics for $Server" Stop-Process -Id $DupViPullStatisticsProcessId -Force } else { AltAndCatchFire "ViPullStatistics for $Server is already running!" @@ -214,20 +220,20 @@ try { if ($SessionFile) { try { $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop - Write-Host "$((Get-Date).ToString("o")) [INFO] SessionToken found in SessionFile, attempting connection to $Server ..." + SexiLogger "[INFO] SessionToken found in SessionFile, attempting connection to $Server ..." # https://zhengwu.org/validating-connection-result-of-connect-viserver/ $ServerConnection = Connect-VIServer -Server $Server -Session $SessionToken -Force -ErrorAction Stop if ($ServerConnection.IsConnected) { # $PwCliContext = Get-PowerCLIContext - Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to vCenter $($ServerConnection.Name) version $($ServerConnection.Version) build $($ServerConnection.Build)" + SexiLogger "[INFO] Connected to vCenter $($ServerConnection.Name) version $($ServerConnection.Version) build $($ServerConnection.Build)" } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] SessionToken not found, invalid or connection failure" - Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting explicit connection ..." + SexiLogger "[WARN] SessionToken not found, invalid or connection failure" + SexiLogger "[WARN] Attempting explicit connection ..." } if (!$($global:DefaultVIServer)) { - ### XXX The session is not authenticated + ### The session is not authenticated # https://github.com/guyrleech/VMware/blob/master/VMware%20GUI.ps1#L2940 try { $createstorexml = New-Object -TypeName XML @@ -243,7 +249,7 @@ if ($SessionFile) { $ServerConnection = Connect-VIServer -Server $Server -User $CredStoreLogin -Password $CredStorePassword -Force -ErrorAction Stop if ($ServerConnection.IsConnected) { # $PwCliContext = Get-PowerCLIContext - Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to vCenter $($ServerConnection.Name) version $($ServerConnection.Version) build $($ServerConnection.Build)" + SexiLogger "[INFO] Connected to vCenter $($ServerConnection.Name) version $($ServerConnection.Version) build $($ServerConnection.Build)" $SessionSecretName = "vmw_" + $Server.Replace(".","_") + ".key" $ServerConnection.SessionSecret | Out-File -FilePath /tmp/$SessionSecretName -Force } @@ -257,7 +263,7 @@ if ($SessionFile) { try { if ($($global:DefaultVIServer)) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing vCenter/ESX $Server ..." + SexiLogger "[INFO] Start processing vCenter/ESX $Server ..." $ServiceInstance = Get-View ServiceInstance -Server $Server # $ServiceManager = Get-View $ServiceInstance.Content.serviceManager -property "" -Server $Server $ServiceInstanceServerClock = $ServiceInstance.CurrentTime() @@ -271,14 +277,30 @@ try { try { if ($ServiceInstance) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing SessionManager & EventManager ..." - $SessionManager = Get-View $ServiceInstance.Content.SessionManager -Property SessionList -Server $Server # Permission to perform this operation was denied. Required privilege 'Sessions.TerminateSession' on managed object with id 'Folder-group-d1'. - $EventManager = Get-View $ServiceInstance.Content.EventManager -Property latestEvent, description -Server $Server + SexiLogger "[INFO] Processing SessionManager & EventManager ..." + try { + $AuthorizationManager = Get-View $ServiceInstance.Content.AuthorizationManager -Server $Server + $group_d1 = Get-View $ServiceInstance.Content.RootFolder + $UserRoleId = $($group_d1.Permission|?{$_.Principal -eq $ServerConnection.User}).RoleId + $UserRole = $AuthorizationManager.RoleList|?{$_.RoleId -eq $UserRoleId} + # $UserTerminateSessionCheck = $AuthorizationManager.HasUserPrivilegeOnEntities($ServiceInstance.Content.RootFolder,$ServerConnection.User,"Sessions.TerminateSession") + if ($UserRole.Privilege -match "Sessions.TerminateSession") { + $SessionManager = Get-View $ServiceInstance.Content.SessionManager -Property SessionList -Server $Server # Permission to perform this operation was denied. Required privilege 'Sessions.TerminateSession' on managed object with id 'Folder-group-d1'. + } else { + SexiLogger "[INFO] Sessions.TerminateSession privilege not detected, SessionManager skipped ..." + } + $EventManager = Get-View $ServiceInstance.Content.EventManager -Property latestEvent, description -Server $Server + if ($vSanPull = Test-Path -Path $("/etc/cron.d/vsan_" + $Server.Replace(".","_"))) { + SexiLogger "[INFO] vSAN collector enabled ..." + } + } catch { + AltAndCatchFire "AuthorizationManager or SessionManager check failure" + } } else { AltAndCatchFire "ServiceInstance check failure" } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] $($Error[0])" } try { @@ -306,14 +328,14 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_version = NameCleaner $($ServiceInstance.Content.About.Version + "." + $ServiceInstance.Content.About.Build) $vmware_version_h["vi.$vcenter_name.vi.version.vpx.product.$vcenter_version"] ++ - Write-Host "$((Get-Date).ToString("o")) [INFO] vCenter objects collect ..." + SexiLogger "[INFO] vCenter objects collect ..." try { $vcenter_folders = Get-View -ViewType Folder -Property Name, Parent -Server $Server $vcenter_datacenters = Get-View -ViewType Datacenter -Property Name, Parent -Server $Server $vcenter_resource_pools = Get-View -ViewType ResourcePool -Property Vm, Parent, Owner, summary.quickStats -Server $Server $vcenter_clusters = Get-View -ViewType ComputeResource -Property name, parent, summary, resourcePool, host, datastore, ConfigurationEx -Server $Server - $vcenter_vmhosts = Get-View -ViewType HostSystem -Property config.network.pnic, config.network.vnic, config.network.dnsConfig.hostName, runtime.connectionState, summary.hardware.numCpuCores, summary.quickStats.distributedCpuFairness, summary.quickStats.distributedMemoryFairness, summary.quickStats.overallCpuUsage, summary.quickStats.overallMemoryUsage, summary.quickStats.uptime, overallStatus, config.storageDevice.hostBusAdapter, vm, name, summary.runtime.healthSystemRuntime.systemHealthInfo.numericSensorInfo, config.product.version, config.product.build, summary.hardware.vendor, summary.hardware.model, summary.hardware.cpuModel, summary.hardware.NumCpuPkgs, Config.VsanHostConfig.ClusterInfo -filter @{"Runtime.ConnectionState" = "^connected$"} -Server $Server + $vcenter_vmhosts = Get-View -ViewType HostSystem -Property config.network.pnic, config.network.vnic, config.network.dnsConfig.hostName, runtime.connectionState, summary.hardware.numCpuCores, summary.quickStats.distributedCpuFairness, summary.quickStats.distributedMemoryFairness, summary.quickStats.overallCpuUsage, summary.quickStats.overallMemoryUsage, summary.quickStats.uptime, overallStatus, config.storageDevice.hostBusAdapter, vm, name, summary.runtime.healthSystemRuntime.systemHealthInfo.numericSensorInfo, config.product.version, config.product.build, summary.hardware.vendor, summary.hardware.model, summary.hardware.cpuModel, summary.hardware.NumCpuPkgs, Config.VsanHostConfig.ClusterInfo, Config.MultipathState -filter @{"Runtime.ConnectionState" = "^connected$"} -Server $Server $vcenter_datastores = Get-View -ViewType Datastore -Property summary, iormConfiguration.enabled, iormConfiguration.statsCollectionEnabled, host -filter @{"summary.accessible" = "true"} -Server $Server $vcenter_pods = Get-View -ViewType StoragePod -Property name, summary, parent, childEntity -Server $Server $vcenter_vms = Get-View -ViewType VirtualMachine -Property name, runtime.maxCpuUsage, runtime.maxMemoryUsage, summary.quickStats.overallCpuUsage, summary.quickStats.overallCpuDemand, summary.quickStats.hostMemoryUsage, summary.quickStats.guestMemoryUsage, summary.quickStats.balloonedMemory, summary.quickStats.compressedMemory, summary.quickStats.swappedMemory, summary.storage.committed, summary.storage.uncommitted, config.hardware.numCPU, layoutEx.file, snapshot, runtime.host, summary.runtime.connectionState, summary.runtime.powerState, summary.config.numVirtualDisks, config.version, config.guestId, config.tools.toolsVersion, summary.quickStats.privateMemory, summary.quickStats.consumedOverheadMemory, summary.quickStats.sharedMemory -filter @{"Summary.Runtime.ConnectionState" = "^connected$"} -Server $Server @@ -321,7 +343,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { AltAndCatchFire "Get-View failure" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Building objects tables ..." + SexiLogger "[INFO] Building objects tables ..." $vcenter_resource_pools_h = @{} $vcenter_resource_pools_owner_vms_h = @{} @@ -427,7 +449,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $script:xfolders_vcenter_parent_h = @{} $script:xfolders_vcenter_type_h = @{} - Write-Host "$((Get-Date).ToString("o")) [INFO] vCenter objects relationship discover ..." + SexiLogger "[INFO] vCenter objects relationship discover ..." foreach ($vcenter_xfolder in [array]$vcenter_datacenters + [array]$vcenter_folders + [array]$vcenter_clusters + [array]$vcenter_resource_pools + [array]$vcenter_vmhosts + [array]$vcenter_pods) { try { @@ -437,7 +459,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } catch {} } - Write-Host "$((Get-Date).ToString("o")) [INFO] Performance metrics collection ..." + SexiLogger "[INFO] Performance metrics collection ..." $HostMultiMetrics = @( "net.bytesRx.average", @@ -449,8 +471,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { "storageAdapter.read.average", "storageAdapter.write.average", "power.power.average", - "datastore.datastoreVMObservedLatency.latest", - # "datastore.totalWriteLatency.average", + "datastore.sizeNormalizedDatastoreLatency.average", + "datastore.totalWriteLatency.average", # "datastore.totalReadLatency.average", "datastore.numberWriteAveraged.average", "datastore.numberReadAveraged.average", @@ -459,15 +481,21 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { "mem.totalCapacity.average" ) - try { - $HostMultiStatsTime = Measure-Command {$HostMultiStats = MultiQueryPerfAll $($vcenter_vmhosts.moref) $HostMultiMetrics} - Write-Host "$((Get-Date).ToString("o")) [INFO] All hosts multi metrics collected in $($HostMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" - } catch { - AltAndCatchFire "ESX MultiQueryPerfAll failure" + if (($vcenter_vmhosts|Measure-Object).Count -gt 0) { + try { + $HostMultiStatsTime = Measure-Command {$HostMultiStats = MultiQueryPerfAll $($vcenter_vmhosts.moref) $HostMultiMetrics} + SexiLogger "[INFO] All hosts multi metrics collected in $($HostMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + } catch { + AltAndCatchFire "ESX MultiQueryPerfAll failure" + } + } else { + SexiLogger "[INFO] No ESX in vCenter $vcenter_name" } - if (($vcenter_vms|Measure-Object).Count -gt 10000) { - Write-Host "$((Get-Date).ToString("o")) [INFO] 10K+ VMs mode for vCenter $vcenter_name" + if (($vcenter_vms|Measure-Object).Count -eq 0) { + SexiLogger "[INFO] No VMs in vCenter $vcenter_name" + } elseif (($vcenter_vms|Measure-Object).Count -gt 10000) { + SexiLogger "[INFO] 10K+ VMs mode for vCenter $vcenter_name" $VmMultiMetricsR1 = @( "cpu.ready.summation", @@ -478,7 +506,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $VmMultiStatsTime = Measure-Command {$VmMultiStats = MultiQueryPerf $($vcenter_vms.moref) $VmMultiMetricsR1} - Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics 1st round collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + SexiLogger "[INFO] All vms multi metrics 1st round collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { AltAndCatchFire "VM MultiQueryPerf R1 failure" } @@ -492,7 +520,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerf $($vcenter_vms.moref) $VmMultiMetricsR2} - Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics 2nd round collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + SexiLogger "[INFO] All vms multi metrics 2nd round collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { AltAndCatchFire "VM MultiQueryPerf R2 failure" } @@ -500,13 +528,13 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # $VmMultiMetricsAll = @( # "virtualdisk.numberWriteAveraged.average", # "virtualdisk.numberReadAveraged.average", - # "net.received.average", - # "net.transmitted.average" + # "net.packetsRx.summation", + # "net.packetsTx.summation" # ) # try { # $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} - # Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + # SexiLogger "[INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" # } catch { # AltAndCatchFire "VM MultiQueryPerfAll failure" # } @@ -525,44 +553,42 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $VmMultiStatsTime = Measure-Command {$VmMultiStats = MultiQueryPerf $($vcenter_vms.moref) $VmMultiMetrics} - Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + SexiLogger "[INFO] All vms multi metrics collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { AltAndCatchFire "VM MultiQueryPerf failure" } - # $VmMultiMetricsAll = @( - # "virtualdisk.numberWriteAveraged.average", - # "virtualdisk.numberReadAveraged.average", - # "net.received.average", - # "net.transmitted.average" - # ) + $VmMultiMetricsAll = @( + "virtualdisk.numberWriteAveraged.average", + "virtualdisk.numberReadAveraged.average" + ) - # try { - # $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} - # Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" - # } catch { - # AltAndCatchFire "VM MultiQueryPerfAll failure" - # } + try { + $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} + SexiLogger "[INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + } catch { + AltAndCatchFire "VM MultiQueryPerfAll failure" + } } - if ($vcenter_clusters_h.Keys) { + if ($vcenter_clusters_h.keys.count -gt 0) { $ClusterMultiMetrics = @( "vmop.numSVMotion.latest", "vmop.numXVMotion.latest" ) try { $ClusterMultiStatsTime = Measure-Command {$ClusterMultiStats = MultiQueryPerf300 $($vcenter_clusters_h.Values.moref) $ClusterMultiMetrics} - Write-Host "$((Get-Date).ToString("o")) [INFO] All Clusters multi metrics collected in $($ClusterMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + SexiLogger "[INFO] All Clusters multi metrics collected in $($ClusterMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" - Write-Host "$((Get-Date).ToString("o")) [EROR] VM MultiQueryPerf failure" + SexiLogger "[EROR] $($Error[0])" + SexiLogger "[EROR] VM MultiQueryPerf300 failure" } } if ($ServiceInstance.Content.About.ApiVersion -ge 6.7) { if ($vcenter_vmhosts_NodeUuid_name_h.Count -gt 0 -and $vSanPull) { - Write-Host "$((Get-Date).ToString("o")) [INFO] vCenter ApiVersion is 6.7+ so we can call vSAN API" + SexiLogger "[INFO] vCenter ApiVersion is 6.7+ so we can call vSAN API" $VsanPerformanceManager = Get-VSANView -Id VsanPerformanceManager-vsan-performance-manager -Server $Server $VsanClusterHealthSystem = Get-VSANView -Id VsanVcClusterHealthSystem-vsan-cluster-health-system -Server $Server $VsanSpaceReportSystem = Get-VSANView -Id VsanSpaceReportSystem-vsan-cluster-space-report-system -Server $Server @@ -570,7 +596,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # VsanClusterHealthSummary/VsanPhysicalDiskHealthSummary better than VsanManagedDisksInfo since we can query at the cluster level # if ($vcenter_clusters_vsan_efa_h.keys) { # $VsanVcDiskManagementSystem = Get-VSANView -Id VimClusterVsanVcDiskManagementSystem-vsan-disk-management-system -Server $Server - # Write-Host "$((Get-Date).ToString("o")) [INFO] vSAN EFA clusters detected ..." + # SexiLogger "[INFO] vSAN EFA clusters detected ..." # } } } @@ -581,66 +607,111 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster = $vcenter_clusters_h[$vcenter_cluster_moref] $vcenter_cluster_name = nameCleaner $vcenter_cluster.Name $vcenter_cluster_dc_name = nameCleaner $(getRootDc $vcenter_cluster) - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name in datacenter $vcenter_cluster_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name in datacenter $vcenter_cluster_dc_name" } catch { AltAndCatchFire "cluster name cleaning issue" } $vcenter_cluster_h = @{} - try { - $vcenter_cluster_pool_quickstats = $vcenter_resource_pools_h[$vcenter_cluster.moref.value].summary.quickStats - - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.ballooned", $vcenter_cluster_pool_quickstats.balloonedMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.compressed", $vcenter_cluster_pool_quickstats.compressedMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.consumedOverhead", $vcenter_cluster_pool_quickstats.consumedOverheadMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.guest", $vcenter_cluster_pool_quickstats.guestMemoryUsage) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.usage", $vcenter_cluster_pool_quickstats.hostMemoryUsage) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.demand", $vcenter_cluster_pool_quickstats.overallCpuDemand) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.usage", $vcenter_cluster_pool_quickstats.overallCpuUsage) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.private", $vcenter_cluster_pool_quickstats.privateMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.shared", $vcenter_cluster_pool_quickstats.sharedMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.swapped", $vcenter_cluster_pool_quickstats.swappedMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.effective", $vcenter_cluster.summary.effectiveMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.total", $vcenter_cluster.summary.totalMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.effective", $vcenter_cluster.summary.effectiveCpu) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.total", $vcenter_cluster.summary.totalCpu) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numVmotions", $vcenter_cluster.summary.numVmotions) - - if($ClusterMultiStats[$PerfCounterTable["vmop.numSVMotion.latest"]][$vcenter_cluster.moref.value][""]) { - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numSVMotions", $ClusterMultiStats[$PerfCounterTable["vmop.numSVMotion.latest"]][$vcenter_cluster.moref.value][""]) + if ($vcenter_resource_pools_h[$vcenter_cluster.moref.value]) { + try { + $vcenter_cluster_pool_quickstats = $vcenter_resource_pools_h[$vcenter_cluster.moref.value].summary.quickStats + + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.ballooned", $vcenter_cluster_pool_quickstats.balloonedMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.compressed", $vcenter_cluster_pool_quickstats.compressedMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.consumedOverhead", $vcenter_cluster_pool_quickstats.consumedOverheadMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.guest", $vcenter_cluster_pool_quickstats.guestMemoryUsage) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.usage", $vcenter_cluster_pool_quickstats.hostMemoryUsage) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.demand", $vcenter_cluster_pool_quickstats.overallCpuDemand) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.usage", $vcenter_cluster_pool_quickstats.overallCpuUsage) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.private", $vcenter_cluster_pool_quickstats.privateMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.shared", $vcenter_cluster_pool_quickstats.sharedMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.swapped", $vcenter_cluster_pool_quickstats.swappedMemory) + + if ($vcenter_cluster_pool_quickstats.overallCpuUsage -gt 0 -and $vcenter_cluster.summary.effectiveCpu -gt 0) { + $vcenter_cluster_pool_quickstats_cpu = $vcenter_cluster_pool_quickstats.overallCpuUsage * 100 / $vcenter_cluster.summary.effectiveCpu + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.cpu.utilization", $vcenter_cluster_pool_quickstats_cpu) + } + + if ($vcenter_cluster_pool_quickstats.hostMemoryUsage -gt 0 -and $vcenter_cluster.summary.effectiveMemory -gt 0) { + $vcenter_cluster_pool_quickstats_ram = $vcenter_cluster_pool_quickstats.hostMemoryUsage * 100 / $vcenter_cluster.summary.effectiveMemory + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.mem.utilization", $vcenter_cluster_pool_quickstats_ram) + } + + } catch { + SexiLogger "[EROR] Cluster $vcenter_cluster_name quickstats collect issue" + SexiLogger "[EROR] $($Error[0])" } + } else { + SexiLogger "[EROR] Cluster $vcenter_cluster_name root resource pool not found ?!" + } - if($ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value][""]) { - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numXVMotions", $ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value][""]) + if ($vcenter_cluster.summary) { + try { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.effective", $vcenter_cluster.summary.effectiveMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.total", $vcenter_cluster.summary.totalMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.effective", $vcenter_cluster.summary.effectiveCpu) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.total", $vcenter_cluster.summary.totalCpu) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numVmotions", $vcenter_cluster.summary.numVmotions) + } catch { + SexiLogger "[EROR] Cluster $vcenter_cluster_name summary collect issue" + SexiLogger "[EROR] $($Error[0])" } + } else { + SexiLogger "[EROR] Cluster $vcenter_cluster_name summary missing " + } - if ($vcenter_cluster_pool_quickstats.overallCpuUsage -gt 0 -and $vcenter_cluster.summary.effectiveCpu -gt 0) { - $vcenter_cluster_pool_quickstats_cpu = $vcenter_cluster_pool_quickstats.overallCpuUsage * 100 / $vcenter_cluster.summary.effectiveCpu - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.cpu.utilization", $vcenter_cluster_pool_quickstats_cpu) + try { + if($ClusterMultiStats[$PerfCounterTable["vmop.numSVMotion.latest"]][$vcenter_cluster.moref.value] -and $ClusterMultiStats[$PerfCounterTable["vmop.numSVMotion.latest"]][$vcenter_cluster.moref.value][""]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numSVMotions", $ClusterMultiStats[$PerfCounterTable["vmop.numSVMotion.latest"]][$vcenter_cluster.moref.value][""]) } - if ($vcenter_cluster_pool_quickstats.hostMemoryUsage -gt 0 -and $vcenter_cluster.summary.effectiveMemory -gt 0) { - $vcenter_cluster_pool_quickstats_ram = $vcenter_cluster_pool_quickstats.hostMemoryUsage * 100 / $vcenter_cluster.summary.effectiveMemory - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.mem.utilization", $vcenter_cluster_pool_quickstats_ram) + if($ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value] -and $ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value][""]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numXVMotions", $ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value][""]) } + } catch { + SexiLogger "[EROR] Cluster $vcenter_cluster_name xyzMotion collect issue" + SexiLogger "[EROR] $($Error[0])" + } + try { if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket -and $vcenter_cluster.summary.DrsScore) { # https://vdc-download.vmware.com/vmwb-repository/dcr-public/bf660c0a-f060-46e8-a94d-4b5e6ffc77ad/208bc706-e281-49b6-a0ce-b402ec19ef82/SDK/vsphere-ws/docs/ReferenceGuide/vim.ClusterComputeResource.Summary.html#numVmsPerDrsScoreBucket - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.0_20", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[0]) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.21_40", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[1]) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.41_60", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[2]) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.61_80", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[3]) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.81_100", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[4]) + if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket[0]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.0_20", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[0]) + } else { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.0_20", 0) + } + if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket[1]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.21_40", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[1]) + } else { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.21_40", 0) + } + if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket[2]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.41_60", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[2]) + } else { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.41_60", 0) + } + if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket[3]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.61_80", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[3]) + } else { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.61_80", 0) + } + if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket[4]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.81_100", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[4]) + } else { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.81_100", 0) + } + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.DrsScore", $vcenter_cluster.summary.DrsScore) } - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name root resource pool not found ?!" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Cluster $vcenter_cluster_name DrsScore collect issue" + SexiLogger "[EROR] $($Error[0])" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name hosts in datacenter $vcenter_cluster_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name hosts in datacenter $vcenter_cluster_dc_name" $vcenter_cluster_hosts_pcpus = 0 $vcenter_cluster_hosts_vms_moref = @() @@ -651,11 +722,13 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_hosts_hba_bytesWrite = 0 $vcenter_cluster_hosts_power_usage = 0 $vcenter_cluster_hosts_vms_dead = 0 - + $vcenter_cluster_hosts_h = @{} foreach ($vcenter_cluster_host in $vcenter_vmhosts_h[$vcenter_cluster.Host.value]|?{$_}) { - $vcenter_cluster_host_name = $vcenter_cluster_host.config.network.dnsConfig.hostName.ToLower() ### why not $vcenter_cluster_host.name.split(".")[0].ToLower() ? + $vcenter_cluster_hosts_h.add($vcenter_cluster_host.moref.value,$vcenter_cluster_host) + + $vcenter_cluster_host_name = $vcenter_cluster_host.config.network.dnsConfig.hostName.ToLower() ### to avoid esx registered by ip if ($vcenter_cluster_host_name -match "localhost") { $vcenter_cluster_host_name = NameCleaner $vcenter_cluster_host.name ### previously vmk0 ip cleaned @@ -668,7 +741,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($vcenter_cluster_host_real_vm_count -gt $vcenter_cluster_host_connected_vm_count) { $vcenter_cluster_hosts_vms_dead += $vcenter_cluster_host_real_vm_count - $vcenter_cluster_host_connected_vm_count } - ### XXX use $vcenter_resource_pools_owner_vms_h + ### TODO use $vcenter_resource_pools_owner_vms_h ? } if ($vcenter_cluster_host.config.product.version -and $vcenter_cluster_host.config.product.build -and $vcenter_cluster_host.summary.hardware.cpuModel -and $vcenter_cluster_host.summary.hardware.NumCpuPkgs) { @@ -697,8 +770,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host sensors issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host sensors issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -714,7 +787,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_hosts_net_bytesTx += $vcenter_cluster_host_vmnic_bytesTx $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.net.$vcenter_cluster_host_vmnic_name.bytesRx", $vcenter_cluster_host_vmnic_bytesRx) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.net.$vcenter_cluster_host_vmnic_name.bytesTx", $vcenter_cluster_host_vmnic_bytesTx) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.net.$vcenter_cluster_host_vmnic_name.linkSpeed", $vcenter_cluster_host_vmnic.linkSpeed.speedMb) ### XXX Still usefull? + # $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.net.$vcenter_cluster_host_vmnic_name.linkSpeed", $vcenter_cluster_host_vmnic.linkSpeed.speedMb) } $vcenter_cluster_host_vmnic_droppedRx = $HostMultiStats[$PerfCounterTable["net.droppedRx.summation"]][$vcenter_cluster_host.moref.value][$vcenter_cluster_host_vmnic_name] @@ -741,12 +814,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name network metrics issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name network metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { - foreach ($vcenter_cluster_host_vmhba in $vcenter_cluster_host.config.storageDevice.hostBusAdapter) { ### XXX dead paths from config.storageDevice.HostBusAdapter to add + foreach ($vcenter_cluster_host_vmhba in $vcenter_cluster_host.config.storageDevice.hostBusAdapter) { $vcenter_cluster_host_vmhba_name = $vcenter_cluster_host_vmhba.device $vcenter_cluster_host_vmhba_bytesRead = $HostMultiStats[$PerfCounterTable["storageAdapter.read.average"]][$vcenter_cluster_host.moref.value][$vcenter_cluster_host_vmhba_name] $vcenter_cluster_host_vmhba_bytesWrite = $HostMultiStats[$PerfCounterTable["storageAdapter.write.average"]][$vcenter_cluster_host.moref.value][$vcenter_cluster_host_vmhba_name] @@ -759,52 +832,96 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name hba metrics issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name hba metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" + } + + try { + foreach ($vcenter_cluster_host_path in $vcenter_cluster_host.Config.MultipathState.Path) { + $vcenter_cluster_h["vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.PathState.$($vcenter_cluster_host_path.PathState)"] ++ + } + } catch { + SexiLogger "[EROR] ESX $vcenter_cluster_host_name MultipathState issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { - $vcenter_cluster_host_power = $HostMultiStats[$PerfCounterTable["power.power.average"]][$vcenter_cluster_host.moref.value][""] - if ($vcenter_cluster_host_power -ge 0) { - $vcenter_cluster_hosts_power_usage += $vcenter_cluster_host_power - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.power", $vcenter_cluster_host_power) + if ($HostMultiStats[$PerfCounterTable["power.power.average"]][$vcenter_cluster_host.moref.value]) { + $vcenter_cluster_host_power = $HostMultiStats[$PerfCounterTable["power.power.average"]][$vcenter_cluster_host.moref.value][""] + if ($vcenter_cluster_host_power -ge 0) { + $vcenter_cluster_hosts_power_usage += $vcenter_cluster_host_power + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.power", $vcenter_cluster_host_power) + } } + } catch { + SexiLogger "[EROR] ESX $vcenter_cluster_host_name fatstats power metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" + } - $vcenter_cluster_host_cpu_totalCapacity = $HostMultiStats[$PerfCounterTable["cpu.totalCapacity.average"]][$vcenter_cluster_host.moref.value][""] - if ($vcenter_cluster_host_cpu_totalCapacity -ge 0 -and $vcenter_cluster_host.summary.quickStats.overallCpuUsage -ge 0) { - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.overallCpuUtilization", $($vcenter_cluster_host.summary.quickStats.overallCpuUsage * 100 / $vcenter_cluster_host_cpu_totalCapacity)) + try { + if ($HostMultiStats[$PerfCounterTable["cpu.totalCapacity.average"]][$vcenter_cluster_host.moref.value]) { + $vcenter_cluster_host_cpu_totalCapacity = $HostMultiStats[$PerfCounterTable["cpu.totalCapacity.average"]][$vcenter_cluster_host.moref.value][""] + if ($vcenter_cluster_host_cpu_totalCapacity -ge 0 -and $vcenter_cluster_host.summary.quickStats.overallCpuUsage -ge 0) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.overallCpuUtilization", $($vcenter_cluster_host.summary.quickStats.overallCpuUsage * 100 / $vcenter_cluster_host_cpu_totalCapacity)) + } } + } catch { + SexiLogger "[EROR] ESX $vcenter_cluster_host_name fatstats cpu.totalCapacity metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" + } - $vcenter_cluster_host_mem_totalCapacity = $HostMultiStats[$PerfCounterTable["mem.totalCapacity.average"]][$vcenter_cluster_host.moref.value][""] - if ($vcenter_cluster_host_cpu_totalCapacity -ge 0 -and $vcenter_cluster_host.summary.quickStats.overallMemoryUsage -ge 0) { - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.overallmemUtilization", $($vcenter_cluster_host.summary.quickStats.overallMemoryUsage * 100 / $vcenter_cluster_host_mem_totalCapacity)) + try { + if ($HostMultiStats[$PerfCounterTable["mem.totalCapacity.average"]][$vcenter_cluster_host.moref.value]) { + $vcenter_cluster_host_mem_totalCapacity = $HostMultiStats[$PerfCounterTable["mem.totalCapacity.average"]][$vcenter_cluster_host.moref.value][""] + if ($vcenter_cluster_host_cpu_totalCapacity -ge 0 -and $vcenter_cluster_host.summary.quickStats.overallMemoryUsage -ge 0) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.overallmemUtilization", $($vcenter_cluster_host.summary.quickStats.overallMemoryUsage * 100 / $vcenter_cluster_host_mem_totalCapacity)) + } } + } catch { + SexiLogger "[EROR] ESX $vcenter_cluster_host_name fatstats mem.totalCapacity metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" + } - $vcenter_cluster_host_cpu_latency = $HostMultiStats[$PerfCounterTable["cpu.latency.average"]][$vcenter_cluster_host.moref.value][""] - if ($vcenter_cluster_host_cpu_latency -ge 0) { - $vcenter_cluster_hosts_cpu_latency += $vcenter_cluster_host_cpu_latency + try { + if ($HostMultiStats[$PerfCounterTable["cpu.latency.average"]][$vcenter_cluster_host.moref.value]) { + $vcenter_cluster_host_cpu_latency = $HostMultiStats[$PerfCounterTable["cpu.latency.average"]][$vcenter_cluster_host.moref.value][""] + if ($vcenter_cluster_host_cpu_latency -ge 0) { + $vcenter_cluster_hosts_cpu_latency += $vcenter_cluster_host_cpu_latency + } } + } catch { + SexiLogger "[EROR] ESX $vcenter_cluster_host_name fatstats cpu.latency metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" + } + try { if ($vcenter_cluster_host.overallStatus.value__) { $vcenter_cluster_host_overallStatus = $vcenter_cluster_host.overallStatus.value__ } else { $vcenter_cluster_host_overallStatus = "0" } + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.overallStatus", $vcenter_cluster_host_overallStatus) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats metrics issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name fatstats overallStatus metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" + } + + try { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedCpuFairness", $vcenter_cluster_host.summary.quickStats.distributedCpuFairness) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedMemoryFairness", $vcenter_cluster_host.summary.quickStats.distributedMemoryFairness) + } catch { + SexiLogger "[EROR] ESX $vcenter_cluster_host_name Distributed Fairness quickstats issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedCpuFairness", $vcenter_cluster_host.summary.quickStats.distributedCpuFairness) ### ToDo - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedMemoryFairness", $vcenter_cluster_host.summary.quickStats.distributedMemoryFairness) ### ToDo $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.overallCpuUsage", $vcenter_cluster_host.summary.quickStats.overallCpuUsage) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.overallMemoryUsage", $vcenter_cluster_host.summary.quickStats.overallMemoryUsage) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.Uptime", $vcenter_cluster_host.summary.quickStats.uptime) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.overallStatus", $vcenter_cluster_host_overallStatus) + } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name quickstats issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name overall quickstats issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } } @@ -830,7 +947,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.power", $vcenter_cluster_hosts_power_usage) } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name vms in datacenter $vcenter_cluster_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name vms in datacenter $vcenter_cluster_dc_name" $vcenter_cluster_vms_vcpus = 0 $vcenter_cluster_vms_vram = 0 @@ -893,7 +1010,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } foreach ($vcenter_cluster_vm_file in $vcenter_cluster_vm_files) { - if(!$vcenter_cluster_vms_files_dedup[$vcenter_cluster_vm_file.name]) { ### XXX would need name & moref + if(!$vcenter_cluster_vms_files_dedup[$vcenter_cluster_vm_file.name]) { ### TODO would need name & moref $vcenter_cluster_vms_files_dedup[$vcenter_cluster_vm_file.name] = $vcenter_cluster_vm_file.size if ($vcenter_cluster_vm_has_snap -and (($vcenter_cluster_vm_file.name -match '-[0-9]{6}-delta\.vmdk') -or ($vcenter_cluster_vm_file.name -match '-[0-9]{6}-sesparse\.vmdk'))) { $vcenter_cluster_vms_files_dedup_total["snapshotExtent"] += $vcenter_cluster_vm_file.size @@ -918,16 +1035,16 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.storage.delta", $vcenter_cluster_vm_snap_size) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $vcenter_cluster_vm_name snapshot compute issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $vcenter_cluster_vm_name snapshot compute issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.storage.committed", $vcenter_cluster_vm.summary.storage.committed) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.storage.uncommitted", $vcenter_cluster_vm.summary.storage.uncommitted) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $vcenter_cluster_vm_name storage commit metric issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $vcenter_cluster_vm_name storage commit metric issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } if ($vcenter_cluster_vm.summary.runtime.powerState -eq "poweredOn") { @@ -997,14 +1114,14 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.diskUsage", 0) } - # if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_cluster_vm.moref.value]) { - # $vcenter_cluster_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.diskiops", $vcenter_cluster_vm_disk_iops) - # } + if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_cluster_vm.moref.value]) { + $vcenter_cluster_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.diskIOPS", $vcenter_cluster_vm_disk_iops) + } - # if ($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$vcenter_cluster_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.received.average"]][$vcenter_cluster_vm.moref.value]) { - # $vcenter_cluster_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.received.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.netUsage", $vcenter_cluster_vm_disk_iops) + # if ($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_cluster_vm.moref.value]) { + # $vcenter_cluster_vm_net_iops = $($($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum) / 300 + # $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.netIOPS", $vcenter_cluster_vm_net_iops) # } if ($VmMultiStats[$PerfCounterTable["net.usage.average"]][$vcenter_cluster_vm.moref.value][""]) { @@ -1048,7 +1165,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.runtime.vm.on", $vcenter_cluster_vms_on) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.runtime.vm.dead", $vcenter_cluster_hosts_vms_dead) - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name datastores in datacenter $vcenter_cluster_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name datastores in datacenter $vcenter_cluster_dc_name" $vcenter_cluster_datastores_count = 0 $vcenter_cluster_datastores_capacity = 0 @@ -1064,9 +1181,9 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_datastores_count ++ - $vcenter_cluster_datastore_hosts = ($vcenter_cluster_datastore.Host|?{$_.mountinfo.Accessible}).key + $vcenter_cluster_datastore_hosts = $vcenter_cluster_hosts_h[$(($vcenter_cluster_datastore.Host|?{$_.mountinfo.Accessible}).key).value].moref - if($vcenter_cluster_datastore.summary.uncommitted -ge 0) { + if ($vcenter_cluster_datastore.summary.uncommitted -ge 0) { $vcenter_cluster_datastore_uncommitted = $vcenter_cluster_datastore.summary.uncommitted } else { $vcenter_cluster_datastore_uncommitted = 0 @@ -1076,27 +1193,34 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.summary.freeSpace", $vcenter_cluster_datastore.summary.freeSpace) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.summary.uncommitted", $vcenter_cluster_datastore_uncommitted) + if ($vcenter_cluster_datastore.summary.capacity -gt 0 -and $vcenter_cluster_datastore.summary.capacity -gt $vcenter_cluster_datastore.summary.freeSpace) { + $vcenter_cluster_datastore_usagepct = ($vcenter_cluster_datastore.summary.capacity - $vcenter_cluster_datastore.summary.freeSpace) * 100 / $vcenter_cluster_datastore.summary.capacity + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.summary.usagePct", $vcenter_cluster_datastore_usagepct) + } + if ($vcenter_cluster_vmdk_per_ds[$vcenter_cluster_datastore_name]) { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.summary.vmdkCount", $vcenter_cluster_vmdk_per_ds[$vcenter_cluster_datastore_name]) } - ### XXX if vmdkCount -gt 0 - $vcenter_cluster_datastores_capacity += $vcenter_cluster_datastore.summary.capacity $vcenter_cluster_datastores_freeSpace += $vcenter_cluster_datastore.summary.freeSpace $vcenter_cluster_datastores_uncommitted += $vcenter_cluster_datastore_uncommitted } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] datastore processing issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] datastore processing issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } if ($vcenter_cluster_datastore.summary.type -notmatch "vsan") { try { $vcenter_cluster_datastore_uuid = $vcenter_cluster_datastore.summary.url.split("/")[-2] - $vcenter_cluster_datastore_latency_raw = $HostMultiStats[$PerfCounterTable["datastore.datastoreVMObservedLatency.latest"]][$vcenter_cluster_datastore_hosts.value]|%{$_[$vcenter_cluster_datastore_uuid]} + $vcenter_cluster_datastore_latency_raw = $($HostMultiStats[$PerfCounterTable["datastore.sizeNormalizedDatastoreLatency.average"]][$vcenter_cluster_datastore_hosts.value])|?{$_.count -gt 0}|%{$_[$vcenter_cluster_datastore_uuid]} #347 $vcenter_cluster_datastore_latency = GetMedian $vcenter_cluster_datastore_latency_raw + if ($vcenter_cluster_datastore_latency -eq 0) { + $vcenter_cluster_datastore_latency_raw = $HostMultiStats[$PerfCounterTable["datastore.totalWriteLatency.average"]][$vcenter_cluster_datastore_hosts.value]|%{$_[$vcenter_cluster_datastore_uuid]} + $vcenter_cluster_datastore_latency = $(GetMedian $vcenter_cluster_datastore_latency_raw) * 1000 + } $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.iorm.sizeNormalizedDatastoreLatency", $vcenter_cluster_datastore_latency) $vcenter_cluster_datastores_latency += $vcenter_cluster_datastore_latency @@ -1106,8 +1230,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.iorm.datastoreIops", $vcenter_cluster_datastore_iops) $vcenter_cluster_datastores_iops += $vcenter_cluster_datastore_iops } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive performance metrics for datastore $vcenter_cluster_datastore_name in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive performance metrics for datastore $vcenter_cluster_datastore_name in cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } } else { # VsanPullStatistics @@ -1117,7 +1241,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($vcenter_cluster_datastore_vsan_cluster_uuid.replace("-","") -match $vcenter_cluster_datastore_vsan_uuid.replace("-","") -and $vSanPull -and $ServiceInstance.Content.About.ApiVersion -ge 6.7) { # skip vSAN HCI Mesh - Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VsanPerfQuery in cluster $vcenter_cluster_name (v6.7+) ..." + SexiLogger "[INFO] Start processing VsanPerfQuery in cluster $vcenter_cluster_name (v6.7+) ..." # https://vdc-download.vmware.com/vmwb-repository/dcr-public/bd51bfdb-3107-4a66-9f63-30aa3fae196e/98507723-ab67-4908-88fa-7c99e0743f0f/vim.cluster.VsanPerformanceManager.html#queryVsanPerf # MethodInvocationException: Exception calling "VsanPerfQueryPerf" with "2" argument(s): "Invalid Argument. Only one wildcard query allowed in query specs." # Config.VsanHostConfig.ClusterInfo.NodeUuid # $VsanPerformanceManager.VsanPerfQueryStatsObjectInformation($vcenter_cluster.moref).VsanHealth @@ -1138,14 +1262,14 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # $VsanHostsAndClusterPerfQuerySpec += New-Object VMware.Vsan.Views.VsanPerfQuerySpec -property @{entityRefId="vsan-vnic-net:*";labels=@("rxThroughput","txThroughput","rxPackets","txPackets","tcpSackRcvBlocksRate","txPacketsLossRate","tcpSackRexmitsRate","rxPacketsLossRate","tcpHalfopenDropRate","tcpRcvoopackRate","tcpRcvduppackRate","tcpRcvdupackRate","tcpTimeoutDropRate","tcpTxRexmitRate","tcpRxErrRate","tcpSackSendBlocksRate");startTime=$ServiceInstanceServerClock_5;endTime=$ServiceInstanceServerClock} # "arpDropRate" if ($vcenter_clusters_vsan_efa_h[$vcenter_cluster_moref]) { # EFA vSan cluster - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing EFA vSAN metrics in cluster $vcenter_cluster_name (v8.0+) ..." + SexiLogger "[INFO] Processing EFA vSAN metrics in cluster $vcenter_cluster_name (v8.0+) ..." # VsanClusterHealthSummary/VsanPhysicalDiskHealthSummary better than VsanManagedDisksInfo since we can query at the cluster level # $VsanHostsAndClusterStoragePoolSpec = New-Object VMware.Vsan.Views.VimVsanHostQueryVsanDisksSpec -property @{vsanDiskType="storagePool"} try { $ClusterHealthSummary = $VsanClusterHealthSystem.VsanQueryVcClusterHealthSummary($vcenter_cluster.moref,$null,$null,$false,"physicalDisksHealth",$true,$null,$null,$null) } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive VsanQueryVcClusterHealthSummary in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive VsanQueryVcClusterHealthSummary in cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } if ($ClusterHealthSummary.PhysicalDisksHealth) { @@ -1165,7 +1289,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $VsanHostsAndClusterPerfQuerySpec += New-Object VMware.Vsan.Views.VsanPerfQuerySpec -property @{entityRefId="vsan-esa-disk-scsifw:$PhysicalDisksHealthVsanUuid";labels=@("latencyDevRead","latencyDevWrite");startTime=$ServiceInstanceServerClock_5;endTime=$ServiceInstanceServerClock} } } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] Empty vSAN PhysicalDisksHealth in cluster $vcenter_cluster_name" + SexiLogger "[WARN] Empty vSAN PhysicalDisksHealth in cluster $vcenter_cluster_name" } } else { # Non EFA vSAN cluster # cache-disk disk-group @@ -1181,11 +1305,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $VsanHostsAndClusterPerfQueryTime = Measure-Command {$VsanHostsAndClusterPerfQuery = $VsanPerformanceManager.VsanPerfQueryPerf($VsanHostsAndClusterPerfQuerySpec,$vcenter_cluster.moref)} - Write-Host "$((Get-Date).ToString("o")) [INFO] VsanPerfQueryPerf metrics collected in $($VsanHostsAndClusterPerfQueryTime.TotalSeconds) sec for vSAN Cluster $vcenter_cluster_name in vCenter $vcenter_name" + SexiLogger "[INFO] VsanPerfQueryPerf metrics collected in $($VsanHostsAndClusterPerfQueryTime.TotalSeconds) sec for vSAN Cluster $vcenter_cluster_name in vCenter $vcenter_name" } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive VsanPerfQuery in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive VsanPerfQuery in cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } if ($VsanHostsAndClusterPerfQuery) { @@ -1318,39 +1442,44 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($vcenter_clusters_vsan_efa_h[$vcenter_cluster_moref]) { # EFA vSan cluster foreach ($PhysicalDisksHealthVsanUuid in $PhysicalDisksHealthVsanUuidHosts.keys) { $PhysicalDisksHealthVsanUuidName = NameCleaner $PhysicalDisksHealthVsanUuidObj[$PhysicalDisksHealthVsanUuid].Name - $PhysicalDisksHealthVsanUuidHost = $vcenter_vmhosts_short_h[$PhysicalDisksHealthVsanUuidHosts[$PhysicalDisksHealthVsanUuid]] - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.percentUsed", $([INT64]$PhysicalDisksHealthVsanUuidObj[$PhysicalDisksHealthVsanUuid].UsedCapacity * 100 / [INT64]$PhysicalDisksHealthVsanUuidObj[$PhysicalDisksHealthVsanUuid].Capacity)) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.latencyDevRead", $VsanPerfEntityMetric[$PhysicalDisksHealthVsanUuid]["vsan-esa-disk-scsifw"]["latencyDevRead"]) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.latencyDevWrite", $VsanPerfEntityMetric[$PhysicalDisksHealthVsanUuid]["vsan-esa-disk-scsifw"]["latencyDevWrite"]) + if ($vcenter_vmhosts_short_h[$PhysicalDisksHealthVsanUuidHosts[$PhysicalDisksHealthVsanUuid]]) { + $PhysicalDisksHealthVsanUuidHost = $vcenter_vmhosts_short_h[$PhysicalDisksHealthVsanUuidHosts[$PhysicalDisksHealthVsanUuid]] + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.percentUsed", $([INT64]$PhysicalDisksHealthVsanUuidObj[$PhysicalDisksHealthVsanUuid].UsedCapacity * 100 / [INT64]$PhysicalDisksHealthVsanUuidObj[$PhysicalDisksHealthVsanUuid].Capacity)) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.latencyDevRead", $VsanPerfEntityMetric[$PhysicalDisksHealthVsanUuid]["vsan-esa-disk-scsifw"]["latencyDevRead"]) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.latencyDevWrite", $VsanPerfEntityMetric[$PhysicalDisksHealthVsanUuid]["vsan-esa-disk-scsifw"]["latencyDevWrite"]) + } + } } else { # Non EFA vSAN cluster foreach ($vcenter_cluster_vsan_Ssd_uuid in $vcenter_clusters_vsan_Ssd_uuid_naa_h[$vcenter_cluster_moref].keys) { - $vcenter_cluster_vsan_Ssd_host = $vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_Ssd_uuid]].name] - $vcenter_cluster_vsan_Ssd_name = $vcenter_clusters_vsan_Ssd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_Ssd_uuid] - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.disk.cache.$vcenter_cluster_vsan_Ssd_name.latencyDevRead", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["latencyDevRead"]) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.disk.cache.$vcenter_cluster_vsan_Ssd_name.latencyDevWrite", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["latencyDevWrite"]) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.domcompmgr.cache.$vcenter_cluster_vsan_Ssd_name.wbFreePct", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["wbFreePct"]) + if ($vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_Ssd_uuid]].name]) { + $vcenter_cluster_vsan_Ssd_host = $vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_Ssd_uuid]].name] + $vcenter_cluster_vsan_Ssd_name = $vcenter_clusters_vsan_Ssd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_Ssd_uuid] + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.disk.cache.$vcenter_cluster_vsan_Ssd_name.latencyDevRead", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["latencyDevRead"]) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.disk.cache.$vcenter_cluster_vsan_Ssd_name.latencyDevWrite", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["latencyDevWrite"]) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.domcompmgr.cache.$vcenter_cluster_vsan_Ssd_name.wbFreePct", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["wbFreePct"]) + } } foreach ($vcenter_cluster_vsan_nonSsd_uuid in $vcenter_clusters_vsan_nonSsd_uuid_naa_h[$vcenter_cluster_moref].keys) { - $vcenter_cluster_vsan_nonSsd_host = $vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_nonSsd_uuid]].name] - $vcenter_cluster_vsan_nonSsd_name = $vcenter_clusters_vsan_nonSsd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_nonSsd_uuid] - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.latencyDevRead", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["latencyDevRead"]) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.latencyDevWrite", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["latencyDevWrite"]) - # $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.capacityUsed", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["capacityUsed"]) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.percentUsed", $([INT64]$VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["capacityUsed"] * 100 / [INT64]$vcenter_vmhosts_vsan_disk_capa_h[$vcenter_cluster_vsan_nonSsd_uuid])) + if ($vcenter_clusters_vsan_nonSsd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_nonSsd_uuid]) { + $vcenter_cluster_vsan_nonSsd_host = $vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_nonSsd_uuid]].name] + $vcenter_cluster_vsan_nonSsd_name = $vcenter_clusters_vsan_nonSsd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_nonSsd_uuid] + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.latencyDevRead", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["latencyDevRead"]) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.latencyDevWrite", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["latencyDevWrite"]) + # $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.capacityUsed", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["capacityUsed"]) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.percentUsed", $([INT64]$VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["capacityUsed"] * 100 / [INT64]$vcenter_vmhosts_vsan_disk_capa_h[$vcenter_cluster_vsan_nonSsd_uuid])) + } } } } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] Empty VsanPerfQuery in cluster $vcenter_cluster_name" + SexiLogger "[WARN] Empty VsanPerfQuery in cluster $vcenter_cluster_name" } - ### XXX add check against multiple vsan datastore on the same cluster - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing SmartStatsSummary in cluster $vcenter_cluster_name (v6.7+) ..." + SexiLogger "[INFO] Start processing SmartStatsSummary in cluster $vcenter_cluster_name (v6.7+) ..." # https://www.virtuallyghetto.com/2017/04/getting-started-wthe-new-powercli-6-5-1-get-vsanview-cmdlet.html # https://github.com/lamw/vghetto-scripts/blob/master/powershell/VSANSmartsData.ps1 $VcClusterSmartStatsSummary = $VsanClusterHealthSystem.VsanQueryVcClusterSmartStatsSummary($vcenter_cluster.moref) @@ -1360,20 +1489,20 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { foreach ($SmartStatsEsxDisk in $SmartStatsEsx.SmartStats) { $SmartStatsEsxDiskName = NameCleaner $SmartStatsEsxDisk.Disk foreach ($SmartStatsEsxDiskStats in $SmartStatsEsxDisk.Stats|?{$_.Value -ne $null}) { - if ($SmartStatsEsxDiskStats.Parameter -and !$vcenter_cluster_h["vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$SmartStatsEsxName.vsan.disk.smart.$SmartStatsEsxDiskName.$($SmartStatsEsxDiskStats.Parameter)"]) { - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$SmartStatsEsxName.vsan.disk.smart.$SmartStatsEsxDiskName.$($SmartStatsEsxDiskStats.Parameter)", $($SmartStatsEsxDiskStats.Value)) + if ($SmartStatsEsxDiskStats.parameter -and !$vcenter_cluster_h["vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$SmartStatsEsxName.vsan.disk.smart.$SmartStatsEsxDiskName.$($SmartStatsEsxDiskStats.parameter)"]) { + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$SmartStatsEsxName.vsan.disk.smart.$SmartStatsEsxDiskName.$($SmartStatsEsxDiskStats.parameter)", $($SmartStatsEsxDiskStats.Value)) } } } } } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive VcClusterSmartStatsSummary in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive VcClusterSmartStatsSummary in cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing spaceUsageByObjectType in vSAN cluster $vcenter_cluster_name (v6.2+) ..." + SexiLogger "[INFO] Processing spaceUsageByObjectType in vSAN cluster $vcenter_cluster_name (v6.2+) ..." $ClusterVsanSpaceUsageReport = $VsanSpaceReportSystem.VsanQuerySpaceUsage($vcenter_cluster.Moref) $ClusterVsanSpaceUsageReportObjList = $ClusterVsanSpaceUsageReport.spaceDetail.spaceUsageByObjectType @@ -1402,12 +1531,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # } # } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive VsanQuerySpaceUsage for cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive VsanQuerySpaceUsage for cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing SyncingVsanObjectsSummary in cluster $vcenter_cluster_name (v6.7+) ..." + SexiLogger "[INFO] Start processing SyncingVsanObjectsSummary in cluster $vcenter_cluster_name (v6.7+) ..." # https://vdc-download.vmware.com/vmwb-repository/dcr-public/b21ba11d-4748-4796-97e2-7000e2543ee1/b4a40704-fbca-4222-902c-2500f5a90f3f/vim.cluster.VsanObjectSystem.html#querySyncingVsanObjectsSummary # https://vdc-download.vmware.com/vmwb-repository/dcr-public/9ab58fbf-b389-4e15-bfd4-a915910be724/7872dcb2-3287-40e1-ba00-71071d0e19ff/vim.vsan.VsanSyncReason.html $QuerySyncingVsanObjectsSummary = $VsanObjectSystem.QuerySyncingVsanObjectsSummary($vcenter_cluster.Moref,$(New-Object VMware.Vsan.Views.VsanSyncingObjectFilter -property @{NumberOfObjects="200"})) @@ -1431,13 +1560,13 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive SyncingVsanObjectsSummary in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive SyncingVsanObjectsSummary in cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VsanObjectIdentityAndHealth in cluster $vcenter_cluster_name ..." - $vcenter_cluster_ObjectIdentities = $VsanObjectSystem.VsanQueryObjectIdentities($vcenter_cluster.moref,$null,$null,$true,$false,$false) ### XXX #ToDo optimize + SexiLogger "[INFO] Start processing VsanObjectIdentityAndHealth in cluster $vcenter_cluster_name ..." + $vcenter_cluster_ObjectIdentities = $VsanObjectSystem.VsanQueryObjectIdentities($vcenter_cluster.moref,$null,$null,$true,$false,$false) ### TODO optimize if ($vcenter_cluster_ObjectIdentities.Health.ObjectHealthDetail) { foreach ($ObjectHealth in $vcenter_cluster_ObjectIdentities.Health.ObjectHealthDetail) { if ($ObjectHealth.NumObjects -gt 0) { @@ -1446,8 +1575,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive VsanObjectIdentityAndHealth from cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive VsanObjectIdentityAndHealth from cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } } } @@ -1466,7 +1595,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.datastore.iops", $vcenter_cluster_datastores_iops) } - Write-Host "$((Get-Date).ToString("o")) [INFO] Sending cluster, hosts, vms and datastores metrics to Graphite for cluster $vcenter_cluster_name ..." + SexiLogger "[INFO] Sending cluster, hosts, vms and datastores metrics to Graphite for cluster $vcenter_cluster_name ..." Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vcenter_cluster_h -DateTime $ExecStart } @@ -1476,7 +1605,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $vcenter_pod_name = NameCleaner $vcenter_pod.Name $vcenter_pod_dc_name = nameCleaner $(getRootDc $vcenter_pod) - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name pod $vcenter_pod_name in datacenter $vcenter_pod_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name pod $vcenter_pod_name in datacenter $vcenter_pod_dc_name" } catch { AltAndCatchFire "pod name cleaning issue" } @@ -1511,7 +1640,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } $vcenter_standalone_host_dc_name = nameCleaner $(getRootDc $vcenter_standalone_pool) - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name in datacenter $vcenter_standalone_host_dc_name" } catch { AltAndCatchFire "standalone_host name cleaning issue" } @@ -1540,8 +1669,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_standalone_host sensors issue in datacenter $vcenter_standalone_host_dc_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_standalone_host sensors issue in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[EROR] $($Error[0])" } if ($vcenter_standalone_host.overallStatus) { @@ -1560,11 +1689,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.quickstats.cpu.total", $vcenter_standalone_pool.summary.totalCpu) $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.quickstats.overallStatus", $vcenter_standalone_host_overallStatus) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_standalone_host_name quickstats issue in datacenter $vcenter_standalone_host_dc_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_standalone_host_name quickstats issue in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[EROR] $($Error[0])" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name datastores in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name datastores in datacenter $vcenter_standalone_host_dc_name" foreach ($vcenter_standalone_host_datastore in $vcenter_datastores_h[$vcenter_standalone_pool.Datastore.value]|?{$_}) { if ($vcenter_standalone_host_datastore.summary.accessible) { @@ -1584,7 +1713,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($vcenter_standalone_host_datastore.summary.type -notmatch "vsan") { $vcenter_standalone_host_datastore_uuid = $vcenter_standalone_host_datastore.summary.url.split("/")[-2] - $vcenter_standalone_host_datastore_latency = $HostMultiStats[$PerfCounterTable["datastore.datastoreVMObservedLatency.latest"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_datastore_uuid] + $vcenter_standalone_host_datastore_latency_raw = $HostMultiStats[$PerfCounterTable["datastore.totalWriteLatency.average"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_datastore_uuid] + $vcenter_standalone_host_datastore_latency = $(GetMedian $vcenter_standalone_host_datastore_latency_raw) * 1000 $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.datastore.$vcenter_standalone_host_datastore_name.iorm.sizeNormalizedDatastoreLatency", $vcenter_standalone_host_datastore_latency) $vcenter_standalone_host_datastore_iops_w = $HostMultiStats[$PerfCounterTable["datastore.numberWriteAveraged.average"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_datastore_uuid] @@ -1593,8 +1723,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.datastore.$vcenter_standalone_host_datastore_name.iorm.datastoreIops", $vcenter_standalone_host_datastore_iops) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] datastore processing issue on ESX $vcenter_standalone_host_name in datacenter $vcenter_standalone_host_dc_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] datastore processing issue on ESX $vcenter_standalone_host_name in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[EROR] $($Error[0])" } } } @@ -1614,12 +1744,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_standalone_host_name network metrics issue in datacenter $vcenter_standalone_host_dc_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_standalone_host_name network metrics issue in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[EROR] $($Error[0])" } try { - foreach ($vcenter_standalone_host_vmhba in $vcenter_standalone_host.config.storageDevice.hostBusAdapter) { ### XXX dead paths from config.storageDevice.HostBusAdapter to add + foreach ($vcenter_standalone_host_vmhba in $vcenter_standalone_host.config.storageDevice.hostBusAdapter) { $vcenter_standalone_host_vmhba_name = $vcenter_standalone_host_vmhba.device $vcenter_standalone_host_vmhba_bytesRead = $HostMultiStats[$PerfCounterTable["storageAdapter.read.average"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_vmhba_name] $vcenter_standalone_host_vmhba_bytesWrite = $HostMultiStats[$PerfCounterTable["storageAdapter.write.average"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_vmhba_name] @@ -1631,11 +1761,20 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_standalone_host_name hba metrics issue in datacenter $vcenter_standalone_host_dc_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_standalone_host_name hba metrics issue in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[EROR] $($Error[0])" + } + + try { + foreach ($vcenter_standalone_host_path in $vcenter_standalone_host.Config.MultipathState.Path) { + $vcenter_standalone_host_h["esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.superstats.PathState.$($vcenter_standalone_host_path.PathState)"] ++ + } + } catch { + SexiLogger "[EROR] ESX $vcenter_standalone_host_name MultipathState issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name vms in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name vms in datacenter $vcenter_standalone_host_dc_name" $StandaloneResourcePoolPrivateMemory = 0 $StandaloneResourcePoolSharedMemory = 0 @@ -1706,7 +1845,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } foreach ($vcenter_standalone_host_vm_file in $vcenter_standalone_host_vm_files) { - if(!$vcenter_standalone_host_vms_files_dedup[$vcenter_standalone_host_vm_file.name]) { ### XXX would need name & moref + if(!$vcenter_standalone_host_vms_files_dedup[$vcenter_standalone_host_vm_file.name]) { ### TODO would need name & moref $vcenter_standalone_host_vms_files_dedup[$vcenter_standalone_host_vm_file.name] = $vcenter_standalone_host_vm_file.size if ($vcenter_standalone_host_vm_has_snap -and (($vcenter_standalone_host_vm_file.name -match '-[0-9]{6}-delta\.vmdk') -or ($vcenter_standalone_host_vm_file.name -match '-[0-9]{6}-sesparse\.vmdk'))) { $vcenter_standalone_host_vms_files_dedup_total["snapshotExtent"] += $vcenter_standalone_host_vm_file.size @@ -1731,16 +1870,16 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.storage.delta", $vcenter_standalone_host_vm_snap_size) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $vcenter_standalone_host_vm_name snapshot compute issue standalone host $vcenter_standalone_host_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $vcenter_standalone_host_vm_name snapshot compute issue standalone host $vcenter_standalone_host_name" + SexiLogger "[EROR] $($Error[0])" } try { $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.storage.committed", $vcenter_standalone_host_vm.summary.storage.committed) $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.storage.uncommitted", $vcenter_standalone_host_vm.summary.storage.uncommitted) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $vcenter_standalone_host_vm_name storage commit metric issue standalone host $vcenter_standalone_host_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $vcenter_standalone_host_vm_name storage commit metric issue standalone host $vcenter_standalone_host_name" + SexiLogger "[EROR] $($Error[0])" } if ($vcenter_standalone_host_vm.summary.runtime.powerState -eq "poweredOn") { @@ -1811,9 +1950,14 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.diskUsage", 0) } - # if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_standalone_host_vm.moref.value]) { - # $vcenter_standalone_host_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.diskiops", $vcenter_standalone_host_vm_disk_iops) + if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_standalone_host_vm.moref.value]) { + $vcenter_standalone_host_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.diskIOPS", $vcenter_standalone_host_vm_disk_iops) + } + + # if ($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_standalone_host_vm.moref.value]) { + # $vcenter_standalone_host_vm_net_iops = $($($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum) / 300 + # $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.netIOPS", $vcenter_standalone_host_vm_net_iops) # } if ($VmMultiStats[$PerfCounterTable["net.usage.average"]][$vcenter_standalone_host_vm.moref.value][""]) { @@ -1874,21 +2018,21 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name SessionList" - - $vcenter_session_list_h = @{} - $SessionList = $SessionManager.sessionList - - if ($SessionList) { - foreach ($vcenter_session in $SessionList) { - $vcenter_session_list_h["vi.$vcenter_name.vi.exec.sessionList.$(NameCleaner $vcenter_session.UserName)"] ++ + if ($SessionManager) { + SexiLogger "[INFO] Processing vCenter $vcenter_name SessionList" + $vcenter_session_list_h = @{} + $SessionList = $SessionManager.sessionList + if ($SessionList) { + foreach ($vcenter_session in $SessionList) { + $vcenter_session_list_h["vi.$vcenter_name.vi.exec.sessionList.$(NameCleaner $vcenter_session.UserName)"] ++ + } + $vcenter_session_list_h["vi.$vcenter_name.vi.exec.sessionCount"] = $($SessionList|Measure-Object).Count + + Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vcenter_session_list_h -DateTime $ExecStart } - $vcenter_session_list_h["vi.$vcenter_name.vi.exec.sessionCount"] = $($SessionList|Measure-Object).Count - - Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vcenter_session_list_h -DateTime $ExecStart } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name events" + SexiLogger "[INFO] Processing vCenter $vcenter_name events" $vcenter_events_h = @{} $vCenterFilteredEventTypeId = [System.Collections.ArrayList]@() @@ -1900,7 +2044,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { foreach ($vCenterEventInfo in $EventManager.Description.EventInfo) { try { if ($vCenterEventInfo.Key -match "EventEx|ExtendedEvent" -and $vCenterEventInfo.fullFormat.split("|")[0] -notmatch "nonviworkload|io\.latency|^esx\.audit\.net\.firewall\.config\.changed") { - if ($vCenterEventInfo.fullFormat.split("|")[0] -match "^esx\.|^com\.vmware\.vc\.ha\.|^com\.vmware\.vc\.HA\.|^vprob\.|^com\.vmware\.vsan\.|^vob\.hbr\.|^com\.vmware\.vcHms\.|^com\.vmware\.vc\.HardwareSensorEvent") { + if ($vCenterEventInfo.fullFormat.split("|")[0] -match "^esx\.|^com\.vmware\.vc\.ha\.|^com\.vmware\.vc\.HA\.|^vprob\.|^com\.vmware\.vsan\.|^com\.vmware\.vc\.vsan\.|^vob\.hbr\.|^com\.vmware\.vcHms\.|^com\.vmware\.vc\.HardwareSensorEvent") { $null = $vCenterFilteredEventTypeId.add($vCenterEventInfo.fullFormat.split("|")[0]) } elseif ($vCenterEventInfo.fullFormat.split("|")[0] -match "^com\.vmware\.vc\." -and $vCenterEventInfo.category -match "warning|error") { $null = $vCenterFilteredEventTypeIdCat.add($vCenterEventInfo.fullFormat.split("|")[0]) @@ -1909,8 +2053,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $null = $vCenterFilteredEventTypeIdCat.add($vCenterEventInfo.key) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] vCenter $vcenter_name EventInfo collect issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] vCenter $vcenter_name EventInfo collect issue" + SexiLogger "[EROR] $($Error[0])" } } @@ -1924,8 +2068,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vCenterEventsHistoryCollector = $EventManager.QueryEvents($vCenterEventFilterSpec) $vCenterEventsHistoryCollectorCat = $EventManager.QueryEvents($vCenterEventFilterSpecCat) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] vCenter $vcenter_name EventManager QueryEvents issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] vCenter $vcenter_name EventManager QueryEvents issue" + SexiLogger "[EROR] $($Error[0])" } if ($vCenterEventsHistoryCollector -or $vCenterEventsHistoryCollectorCat) { @@ -1957,10 +2101,10 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vcenter_events_h -DateTime $ExecStart } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] vCenter $vcenter_name no new event collected" + SexiLogger "[INFO] vCenter $vcenter_name no new event collected" } } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] No EventInfo to process in vCenter $vcenter_name" + SexiLogger "[EROR] No EventInfo to process in vCenter $vcenter_name" } } @@ -1968,11 +2112,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vmware_version_h -DateTime $ExecStart - Write-Host "$((Get-Date).ToString("o")) [INFO] End processing vCenter $vcenter_name" + SexiLogger "[INFO] End processing vCenter $vcenter_name" } elseif ($ServiceInstance.Content.About.ApiType -match "HostAgent") { - Write-Host "$((Get-Date).ToString("o")) [INFO] HostAgent detected, collecting unmanaged objects ..." + SexiLogger "[INFO] HostAgent detected, collecting unmanaged objects ..." $vcenter_name = "_unmanaged_" $unmanaged_host_dc_name = "_unmanaged_" @@ -1990,7 +2134,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { AltAndCatchFire "Get-View failure" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Performance metrics collection ..." + SexiLogger "[INFO] Performance metrics collection ..." $HostMultiMetrics = @( "net.bytesRx.average", @@ -1998,14 +2142,14 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { "storageAdapter.read.average", "storageAdapter.write.average" # "power.power.average", - # "datastore.datastoreVMObservedLatency.latest", + # "datastore.sizeNormalizedDatastoreLatency.latest", # "datastore.numberWriteAveraged.average", # "datastore.numberReadAveraged.average" ) try { $HostMultiStatsTime = Measure-Command {$HostMultiStats = MultiQueryPerfAll $($vcenter_vmhosts.moref) $HostMultiMetrics} - Write-Host "$((Get-Date).ToString("o")) [INFO] All hosts multi metrics collected in $($HostMultiStatsTime.TotalSeconds) sec for Unmanaged ESX $esx_name" + SexiLogger "[INFO] All hosts multi metrics collected in $($HostMultiStatsTime.TotalSeconds) sec for Unmanaged ESX $esx_name" } catch { AltAndCatchFire "ESX MultiQueryPerfAll failure" } @@ -2021,34 +2165,27 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { "net.usage.average" ) - # $VmMultiMetricsAll = @( - # "virtualdisk.numberWriteAveraged.average", - # "virtualdisk.numberReadAveraged.average", - # "net.received.average", - # "net.transmitted.average" - # ) + $VmMultiMetricsAll = @( + "virtualdisk.numberWriteAveraged.average", + "virtualdisk.numberReadAveraged.average" + ) if ($vcenter_vms) { try { $VmMultiStatsTime = Measure-Command {$VmMultiStats = MultiQueryPerf $($vcenter_vms.moref) $VmMultiMetrics} - Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics collected in $($VmMultiStatsTime.TotalSeconds) sec for Unmanaged ESX $esx_name" + SexiLogger "[INFO] All vms multi metrics collected in $($VmMultiStatsTime.TotalSeconds) sec for Unmanaged ESX $esx_name" } catch { AltAndCatchFire "VM MultiQueryPerf failure" } - # try { - # $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} - # Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" - # } catch { - # AltAndCatchFire "VM MultiQueryPerfAll failure" - # } + try { + $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} + SexiLogger "[INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + } catch { + AltAndCatchFire "VM MultiQueryPerfAll failure" + } } - - - - - try { $unmanaged_host = $vcenter_vmhosts $unmanaged_compute_resource = $vcenter_clusters @@ -2058,7 +2195,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $unmanaged_host_name = NameCleaner $unmanaged_host.name ### previously vmk0 ip cleaned } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing Unmanaged ESX $esx_name" + SexiLogger "[INFO] Processing Unmanaged ESX $esx_name" } catch { AltAndCatchFire "Unmanaged ESX name cleaning issue" } @@ -2087,8 +2224,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name sensors issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name sensors issue" + SexiLogger "[EROR] $($Error[0])" } if ($unmanaged_host.overallStatus.value__) { @@ -2107,11 +2244,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.cpu.total", $unmanaged_compute_resource.summary.totalCpu) $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.overallStatus", $unmanaged_host_overallStatus) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name quickstats issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name quickstats issue" + SexiLogger "[EROR] $($Error[0])" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing Unmanaged ESX $esx_name datastores" + SexiLogger "[INFO] Processing Unmanaged ESX $esx_name datastores" foreach ($unmanaged_host_datastore in $vcenter_datastores) { if ($unmanaged_host_datastore.summary.accessible) { @@ -2140,8 +2277,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.datastore.$unmanaged_host_datastore_name.iorm.datastoreIops", $unmanaged_host_datastore_iops) # } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] datastore processing issue on Unmanaged ESX $esx_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] datastore processing issue on Unmanaged ESX $esx_name" + SexiLogger "[EROR] $($Error[0])" } } } @@ -2161,12 +2298,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name network metrics issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name network metrics issue" + SexiLogger "[EROR] $($Error[0])" } try { - foreach ($unmanaged_host_vmhba in $unmanaged_host.config.storageDevice.hostBusAdapter) { ### XXX dead paths from config.storageDevice.HostBusAdapter to add + foreach ($unmanaged_host_vmhba in $unmanaged_host.config.storageDevice.hostBusAdapter) { $unmanaged_host_vmhba_name = $unmanaged_host_vmhba.device $unmanaged_host_vmhba_bytesRead = $HostMultiStats[$PerfCounterTable["storageAdapter.read.average"]][$unmanaged_host.moref.value][$unmanaged_host_vmhba_name] $unmanaged_host_vmhba_bytesWrite = $HostMultiStats[$PerfCounterTable["storageAdapter.write.average"]][$unmanaged_host.moref.value][$unmanaged_host_vmhba_name] @@ -2178,17 +2315,17 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name hba metrics issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name hba metrics issue" + SexiLogger "[EROR] $($Error[0])" } - $UnamagedResourcePoolPrivateMemory = 0 - $UnamagedResourcePoolSharedMemory = 0 - $UnamagedResourcePoolBalloonedMemory = 0 - $UnamagedResourcePoolCompressedMemory = 0 - $UnamagedResourcePoolSwappedMemory = 0 - $UnamagedResourcePoolGuestMemoryUsage = 0 - $UnamagedResourcePoolConsumedOverheadMemory = 0 + $UnmanagedResourcePoolPrivateMemory = 0 + $UnmanagedResourcePoolSharedMemory = 0 + $UnmanagedResourcePoolBalloonedMemory = 0 + $UnmanagedResourcePoolCompressedMemory = 0 + $UnmanagedResourcePoolSwappedMemory = 0 + $UnmanagedResourcePoolGuestMemoryUsage = 0 + $UnmanagedResourcePoolConsumedOverheadMemory = 0 $unmanaged_host_vms_vcpus = 0 $unmanaged_host_vms_vram = 0 @@ -2260,7 +2397,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } foreach ($unmanaged_host_vm_file in $unmanaged_host_vm_files) { - if(!$unmanaged_host_vms_files_dedup[$unmanaged_host_vm_file.name]) { ### XXX would need name & moref + if(!$unmanaged_host_vms_files_dedup[$unmanaged_host_vm_file.name]) { ### TODO would need name & moref $unmanaged_host_vms_files_dedup[$unmanaged_host_vm_file.name] = $unmanaged_host_vm_file.size if ($unmanaged_host_vm_has_snap -and (($unmanaged_host_vm_file.name -match '-[0-9]{6}-delta\.vmdk') -or ($unmanaged_host_vm_file.name -match '-[0-9]{6}-sesparse\.vmdk'))) { $unmanaged_host_vms_files_dedup_total["snapshotExtent"] += $unmanaged_host_vm_file.size @@ -2285,16 +2422,16 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.storage.delta", $unmanaged_host_vm_snap_size) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $unmanaged_host_vm_name snapshot compute issue standalone host $unmanaged_host_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $unmanaged_host_vm_name snapshot compute issue standalone host $unmanaged_host_name" + SexiLogger "[EROR] $($Error[0])" } try { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.storage.committed", $unmanaged_host_vm.summary.storage.committed) $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.storage.uncommitted", $unmanaged_host_vm.summary.storage.uncommitted) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $unmanaged_host_vm_name storage commit metric issue standalone host $unmanaged_host_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $unmanaged_host_vm_name storage commit metric issue standalone host $unmanaged_host_name" + SexiLogger "[EROR] $($Error[0])" } if ($unmanaged_host_vm.summary.runtime.powerState -eq "poweredOn") { @@ -2322,17 +2459,17 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($unmanaged_host_vm.summary.quickStats.balloonedMemory -gt 0) { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.quickstats.BalloonedMemory", $unmanaged_host_vm.summary.quickStats.balloonedMemory) - $UnamagedResourcePoolBalloonedMemory += $unmanaged_host_vm.summary.quickStats.balloonedMemory + $UnmanagedResourcePoolBalloonedMemory += $unmanaged_host_vm.summary.quickStats.balloonedMemory } if ($unmanaged_host_vm.summary.quickStats.compressedMemory -gt 0) { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.quickstats.CompressedMemory", $unmanaged_host_vm.summary.quickStats.compressedMemory) - $UnamagedResourcePoolCompressedMemory += $unmanaged_host_vm.summary.quickStats.compressedMemory + $UnmanagedResourcePoolCompressedMemory += $unmanaged_host_vm.summary.quickStats.compressedMemory } if ($unmanaged_host_vm.summary.quickStats.swappedMemory -gt 0) { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.quickstats.SwappedMemory", $unmanaged_host_vm.summary.quickStats.swappedMemory) - $UnamagedResourcePoolSwappedMemory += $unmanaged_host_vm.summary.quickStats.swappedMemory + $UnmanagedResourcePoolSwappedMemory += $unmanaged_host_vm.summary.quickStats.swappedMemory } if ($VmMultiStats[$PerfCounterTable["cpu.ready.summation"]][$unmanaged_host_vm.moref.value][""]) { @@ -2357,7 +2494,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($VmMultiStats[$PerfCounterTable["disk.maxTotalLatency.latest"]][$unmanaged_host_vm.moref.value][""]) { $unmanaged_host_vm_disk_latency = $VmMultiStats[$PerfCounterTable["disk.maxTotalLatency.latest"]][$unmanaged_host_vm.moref.value][""] $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.maxTotalLatency", $unmanaged_host_vm_disk_latency) - } + }Unmanaged if ($VmMultiStats[$PerfCounterTable["virtualdisk.write.average"]][$unmanaged_host_vm.moref.value][""] -ge 0 -and $VmMultiStats[$PerfCounterTable["virtualdisk.read.average"]][$unmanaged_host_vm.moref.value][""] -ge 0) { $unmanaged_host_vm_disk_usage = $VmMultiStats[$PerfCounterTable["virtualdisk.write.average"]][$unmanaged_host_vm.moref.value][""] + $VmMultiStats[$PerfCounterTable["virtualdisk.read.average"]][$unmanaged_host_vm.moref.value][""] @@ -2366,14 +2503,14 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.diskUsage", 0) } - # if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$unmanaged_host_vm.moref.value]) { - # $unmanaged_host_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.diskiops", $unmanaged_host_vm_disk_iops) - # } + if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$unmanaged_host_vm.moref.value]) { + $unmanaged_host_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.diskIOPS", $unmanaged_host_vm_disk_iops) + } - # if ($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$unmanaged_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.received.average"]][$unmanaged_host_vm.moref.value]) { - # $unmanaged_host_vm_net_usage = $($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.received.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.netUsage", $unmanaged_host_vm_net_usage) + # if ($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$unmanaged_host_vm.moref.value]) { + # $unmanaged_host_vm_net_iops = $($($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum) / 300 + # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.netIOPS", $unmanaged_host_vm_net_iops) # } if ($VmMultiStats[$PerfCounterTable["net.usage.average"]][$unmanaged_host_vm.moref.value][""]) { @@ -2383,14 +2520,14 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.netUsage", 0) } - if ($unmanaged_host_vm.summary.quickStats.privateMemory -gt 0) {$UnamagedResourcePoolPrivateMemory += $unmanaged_host_vm.summary.quickStats.privateMemory} - if ($unmanaged_host_vm.summary.quickStats.GuestMemoryUsage -gt 0) {$UnamagedResourcePoolGuestMemoryUsage += $unmanaged_host_vm.summary.quickStats.GuestMemoryUsage} - if ($unmanaged_host_vm.summary.quickStats.SharedMemory -gt 0) {$UnamagedResourcePoolSharedMemory += $unmanaged_host_vm.summary.quickStats.SharedMemory} - if ($unmanaged_host_vm.summary.quickStats.ConsumedOverheadMemory -gt 0) {$UnamagedResourcePoolConsumedOverheadMemory += $unmanaged_host_vm.summary.quickStats.ConsumedOverheadMemory} + if ($unmanaged_host_vm.summary.quickStats.privateMemory -gt 0) {$UnmanagedResourcePoolPrivateMemory += $unmanaged_host_vm.summary.quickStats.privateMemory} + if ($unmanaged_host_vm.summary.quickStats.GuestMemoryUsage -gt 0) {$UnmanagedResourcePoolGuestMemoryUsage += $unmanaged_host_vm.summary.quickStats.GuestMemoryUsage} + if ($unmanaged_host_vm.summary.quickStats.SharedMemory -gt 0) {$UnmanagedResourcePoolSharedMemory += $unmanaged_host_vm.summary.quickStats.SharedMemory} + if ($unmanaged_host_vm.summary.quickStats.ConsumedOverheadMemory -gt 0) {$UnmanagedResourcePoolConsumedOverheadMemory += $unmanaged_host_vm.summary.quickStats.ConsumedOverheadMemory} } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $unmanaged_host_vm_name metric issue on unmanaged host $unmanaged_host_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $unmanaged_host_vm_name metric issue on unmanaged host $unmanaged_host_name" + SexiLogger "[EROR] $($Error[0])" } } elseif ($unmanaged_host_vm.summary.runtime.powerState -eq "poweredOff") { @@ -2400,25 +2537,25 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } try { - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.ballooned", $UnamagedResourcePoolBalloonedMemory) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.compressed", $UnamagedResourcePoolCompressedMemory) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.consumedOverhead", $UnamagedResourcePoolConsumedOverheadMemory) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.guest", $UnamagedResourcePoolGuestMemoryUsage) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.private", $UnamagedResourcePoolPrivateMemory) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.shared", $UnamagedResourcePoolSharedMemory) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.swapped", $UnamagedResourcePoolSwappedMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.ballooned", $UnmanagedResourcePoolBalloonedMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.compressed", $UnmanagedResourcePoolCompressedMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.consumedOverhead", $UnmanagedResourcePoolConsumedOverheadMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.guest", $UnmanagedResourcePoolGuestMemoryUsage) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.private", $UnmanagedResourcePoolPrivateMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.shared", $UnmanagedResourcePoolSharedMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.swapped", $UnmanagedResourcePoolSwappedMemory) $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.runtime.vm.total", $($unmanaged_host_vms_on + $unmanaged_host_vms_off)) $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.runtime.vm.on", $unmanaged_host_vms_on) $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.runtime.vm.dead", $unmanaged_host_vms_dead) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name quickstats issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name quickstats issue" + SexiLogger "[EROR] $($Error[0])" } Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $unmanaged_host_h -DateTime $ExecStart - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing Unmanaged ESX $esx_name events" + SexiLogger "[INFO] Processing Unmanaged ESX $esx_name events" $vcenter_events_h = @{} $vCenterFilteredEventTypeId = @() @@ -2434,8 +2571,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $null = $vCenterFilteredEventTypeIdCat.add($vCenterEventInfo.key) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name EventInfo collect issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name EventInfo collect issue" + SexiLogger "[EROR] $($Error[0])" } } @@ -2456,8 +2593,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vCenterEventsHistoryCollectorCat = $vCenterEventsHistoryCollectorCatObj.ReadNextEvents("999") } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name EventManager QueryEvents issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name EventManager QueryEvents issue" + SexiLogger "[EROR] $($Error[0])" } if ($vCenterEventsHistoryCollector -or $vCenterEventsHistoryCollectorCat) { @@ -2484,10 +2621,10 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vcenter_events_h -DateTime $ExecStart } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] Unmanaged ESX $esx_name no new event collected" + SexiLogger "[INFO] Unmanaged ESX $esx_name no new event collected" } } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] No EventInfo to process in Unmanaged ESX $esx_name" + SexiLogger "[EROR] No EventInfo to process in Unmanaged ESX $esx_name" } } @@ -2495,7 +2632,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vmware_version_h -DateTime $ExecStart - Write-Host "$((Get-Date).ToString("o")) [INFO] End processing Unmanaged ESX $esx_name" + SexiLogger "[INFO] End processing Unmanaged ESX $esx_name" } else { AltAndCatchFire "$Server is not a vCenter/ESXi!" diff --git a/opt/sexigraf/VsanPullStatistics.ps1 b/opt/sexigraf/VsanPullStatistics.ps1 index d877e91a..5d8e730e 100644 --- a/opt/sexigraf/VsanPullStatistics.ps1 +++ b/opt/sexigraf/VsanPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.79" +$ScriptVersion = "0.9.80" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -98,6 +98,8 @@ try { exit } +AltAndCatchFire "Deprecated script, now integrated to ViPullStatistics" + try { Write-Host "$((Get-Date).ToString("o")) [INFO] Importing PowerCli and Graphite PowerShell modules ..." Import-Module VMware.VimAutomation.Common, VMware.VimAutomation.Core, VMware.VimAutomation.Sdk, VMware.VimAutomation.Storage diff --git a/opt/sexigraf/xMotionGarbageCollector.ps1 b/opt/sexigraf/xMotionGarbageCollector.ps1 new file mode 100644 index 00000000..9736aba0 --- /dev/null +++ b/opt/sexigraf/xMotionGarbageCollector.ps1 @@ -0,0 +1,254 @@ +#!/usr/bin/pwsh -Command +# + +$ScriptVersion = "0.9.3" + +$ErrorActionPreference = "SilentlyContinue" +$WarningPreference = "SilentlyContinue" +(Get-Process -Id $pid).PriorityClass = 'idle' +# $stopwatch = [system.diagnostics.stopwatch]::StartNew() + +function AltAndCatchFire { + Param($ExitReason) + Write-Host "$((Get-Date).ToString("o")) [EROR] $ExitReason" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + Stop-Transcript + exit +} + +try { + Start-Transcript -Path "/var/log/sexigraf/xMotionGarbageCollector.log" -Append -Confirm:$false -Force -UseMinimalHeader + Write-Host "$((Get-Date).ToString("o")) [INFO] xMotionGarbageCollector v$ScriptVersion" +} catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] xMotionGarbageCollector logging failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + exit +} + +try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for another xMotionGarbageCollector ..." + $DupViVmInventoryProcess = Get-PSHostProcessInfo|%{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}|?{$_ -match "xMotionGarbageCollector"} + # https://github.com/PowerShell/PowerShell/issues/13944 + if (($DupViVmInventoryProcess|Measure-Object).Count -gt 1) { + $DupViVmInventoryProcessId = (Get-PSHostProcessInfo|?{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '|?{$_ -match "xMotionGarbageCollector"}}).ProcessId[0] + $DupViVmInventoryProcessTime = [INT32](ps -p $DupViVmInventoryProcessId -o etimes).split()[-1] + if ($DupViVmInventoryProcessTime -gt 21600) { + Write-Host "$((Get-Date).ToString("o")) [WARN] xMotionGarbageCollector is already running for more than 6 hours!" + Write-Host "$((Get-Date).ToString("o")) [WARN] Killing stunned xMotionGarbageCollector" + Stop-Process -Id $DupViVmInventoryProcessId -Force + } else { + AltAndCatchFire "xMotionGarbageCollector is already running!" + } + } +} catch { + AltAndCatchFire "xMotionGarbageCollector process lookup failure" +} + +Write-Host "$((Get-Date).ToString("o")) [INFO] Collecting VM folders ..." + +try { + $VmFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending +} catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Collecting VM folders issue ..." +} + +if ($VmFolders) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xvmotioned vms aka DstVmMigratedEvent ..." + + $VmFolders_h = @{} + $VmFoldersDup_h = @{} + foreach ($VmFolder in $VmFolders) { + if (!$VmFolders_h[$VmFolder.basename]) { + $VmFolders_h.add($VmFolder.basename,@($VmFolder)) + } else { + $VmFolders_h[$VmFolder.basename] += $VmFolder + if (!$VmFoldersDup_h[$VmFolder.basename]) { + $VmFoldersDup_h.add($VmFolder.basename,"1") + } + } + } + + if ($VmFoldersDup_h.Count -gt 0) { + Write-Host "$((Get-Date).ToString("o")) [INFO] $($VmFoldersDup_h.Count) Duplicate vm folders found across clusters, evaluating mobility ..." + foreach ($VmDup in $($VmFoldersDup_h.keys)) { + + if ($VmFolders_h[$VmDup].count -lt 2) { + Write-Host "$((Get-Date).ToString("o")) [EROR] VM $VmDup has less than 2 copies, skipping ..." + continue + } elseif ($VmFolders_h[$VmDup].count -gt 2) { + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has more than 2 copies ..." + } + + $VmDupFolders = $VmFolders_h[$VmDup] + $VmDupSrcDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[1] + $VmDupDstDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[0] + + try { + $VmDupSrcWsp = Get-Item $($VmDupSrcDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $VmDupDstWsp = Get-Item $($VmDupDstDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for vm $VmDup ..." + continue + } + + if (($VmDupDstDir.CreationTimeUtc -gt $VmDupSrcDir.CreationTimeUtc) -and ($VmDupDstWsp.LastWriteTimeUtc - $VmDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { + + $VmDupDstVc = $VmDupDstDir.FullName.split("/")[5] + $VmDupDstDc = $VmDupDstDir.FullName.split("/")[6] + $VmDupDstClu = $VmDupDstDir.FullName.split("/")[7] + $VmDupSrcClu = $VmDupSrcDir.FullName.split("/")[7] + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has been moved from cluster $VmDupSrcClu to cluster $VmDupDstClu a while ago, merging metrics to the new destination if possible ..." + $VmDupWsps2Mv = Get-ChildItem -Recurse $VmDupSrcDir.FullName -Filter *.wsp + foreach ($VmDupWsp2Mv in $VmDupWsps2Mv) { + $WspRelativePath = $($VmDupWsp2Mv.FullName -split "/vm/")[1] + $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $VmDupDstVc + "/" + $VmDupDstDc + "/" + $VmDupDstClu + "/vm/" + $WspRelativePath) + if (Test-Path $DstWspFullPath) { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $DstWspFullPath whisper-info" + $VmDupWspSrcInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $DstWspFullPath" + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $($VmDupWsp2Mv.FullName) whisper-info" + $VmDupWsp2MvInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $($VmDupWsp2Mv.FullName)" + + if (Compare-Object $VmDupWspSrcInfo $VmDupWsp2MvInfo) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Resizing $($VmDupWsp2Mv.FullName) and $DstWspFullPath" + $VmDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $VmDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($VmDupWsp2Mv.FullName) 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + } + + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($VmDupWsp2Mv.FullName) to $DstWspFullPath" + $VmDupWsp2MvMerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($VmDupWsp2Mv.FullName) $DstWspFullPath" + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." + continue + } + } else { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($VmDupWsp2Mv.FullName)" + $VmDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop + $VmDupWspMv = Move-Item $VmDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." + continue + } + } + } + Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($VmDupSrcDir.FullName)" + try { + Remove-Item -Recurse $($VmDupSrcDir.FullName) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($VmDupSrcDir.FullName) issue ..." + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup move is too recent, has clones or has come back to its original location ..." + } + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated vm folders found" + } +} + +Write-Host "$((Get-Date).ToString("o")) [INFO] Collecting ESX folders ..." + +try { + $EsxFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending +} catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Collecting ESX folders issue ..." +} + +if ($EsxFolders) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for *xvmotioned* ESXs aka DstEsxMigratedEvent ..." + + $EsxFolders_h = @{} + $EsxFoldersDup_h = @{} + foreach ($EsxFolder in $EsxFolders) { + if (!$EsxFolders_h[$EsxFolder.basename]) { + $EsxFolders_h.add($EsxFolder.basename,@($EsxFolder)) + } else { + $EsxFolders_h[$EsxFolder.basename] += $EsxFolder + if (!$EsxFoldersDup_h[$EsxFolder.basename]) { + $EsxFoldersDup_h.add($EsxFolder.basename,"1") + } + } + } + + if ($EsxFoldersDup_h.Count -gt 0) { + Write-Host "$((Get-Date).ToString("o")) [INFO] $($EsxFoldersDup_h.Count) Duplicate Esx folders found across clusters, evaluating mobility ..." + foreach ($EsxDup in $($EsxFoldersDup_h.keys)) { + + if ($EsxFolders_h[$EsxDup].count -lt 2) { + Write-Host "$((Get-Date).ToString("o")) [EROR] Esx $EsxDup has less than 2 copies, skipping ..." + continue + } elseif ($EsxFolders_h[$EsxDup].count -gt 2) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has more than 2 copies ..." + } + + $EsxDupFolders = $EsxFolders_h[$EsxDup] + $EsxDupSrcDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[1] + $EsxDupDstDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[0] + + try { + $EsxDupSrcWsp = Get-Item $($EsxDupSrcDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $EsxDupDstWsp = Get-Item $($EsxDupDstDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for Esx $EsxDup ..." + continue + } + + if (($EsxDupDstDir.CreationTimeUtc -gt $EsxDupSrcDir.CreationTimeUtc) -and ($EsxDupDstWsp.LastWriteTimeUtc - $EsxDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { + + $EsxDupDstVc = $EsxDupDstDir.FullName.split("/")[5] + $EsxDupDstDc = $EsxDupDstDir.FullName.split("/")[6] + $EsxDupDstClu = $EsxDupDstDir.FullName.split("/")[7] + $EsxDupSrcClu = $EsxDupSrcDir.FullName.split("/")[7] + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has been moved from cluster $EsxDupSrcClu to cluster $EsxDupDstClu a while ago, merging metrics to the new destination if possible ..." + $EsxDupWsps2Mv = Get-ChildItem -Recurse $EsxDupSrcDir.FullName -Filter *.wsp + foreach ($EsxDupWsp2Mv in $EsxDupWsps2Mv) { + $WspRelativePath = $($EsxDupWsp2Mv.FullName -split "/Esx/")[1] + $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $EsxDupDstVc + "/" + $EsxDupDstDc + "/" + $EsxDupDstClu + "/esx/" + $WspRelativePath) + if (Test-Path $DstWspFullPath) { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $DstWspFullPath whisper-info" + $VmDupWspSrcInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $DstWspFullPath" + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $($EsxDupWsp2Mv.FullName) whisper-info" + $EsxDupWsp2MvInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $($EsxDupWsp2Mv.FullName)" + + if (Compare-Object $VmDupWspSrcInfo $EsxDupWsp2MvInfo) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Resizing $($EsxDupWsp2Mv.FullName) and $DstWspFullPath" + $EsxDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $EsxDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($EsxDupWsp2Mv.FullName) 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + } + + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($EsxDupWsp2Mv.FullName) to $DstWspFullPath" + $EsxDupWsp2MEsxerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($EsxDupWsp2Mv.FullName) $DstWspFullPath" + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." + continue + } + } else { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($EsxDupWsp2Mv.FullName)" + $EsxDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop + $EsxDupWspMv = Move-Item $EsxDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." + continue + } + } + } + Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($EsxDupSrcDir.FullName)" + try { + Remove-Item -Recurse $($EsxDupSrcDir.FullName) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($EsxDupSrcDir.FullName) issue ..." + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup move is too recent, has clones or has come back to its original location ..." + } + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated Esx folders found" + } +} +Write-Host "$((Get-Date).ToString("o")) [INFO] If you get of lot of clones in this run, you should consider the Autopurge feature" +Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf xMotionGarbageCollector has left the building ..." \ No newline at end of file diff --git a/usr/share/grafana/public/dashboards/home.json b/usr/share/grafana/public/dashboards/home.json index 37d2638e..96ff0249 100644 --- a/usr/share/grafana/public/dashboards/home.json +++ b/usr/share/grafana/public/dashboards/home.json @@ -1,5 +1,5 @@ { - "editable": true, + "editable": false, "gnetId": null, "graphTooltip": 1, "id": 62, diff --git a/var/lib/grafana/dashboards/Miscellaneous/FreeNAS.json b/var/lib/grafana/dashboards/Miscellaneous/FreeNAS.json index cf7c0c3e..a453ecd8 100644 --- a/var/lib/grafana/dashboards/Miscellaneous/FreeNAS.json +++ b/var/lib/grafana/dashboards/Miscellaneous/FreeNAS.json @@ -51,7 +51,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -140,7 +140,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -213,7 +213,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -286,7 +286,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -360,7 +360,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -453,7 +453,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -535,7 +535,7 @@ "lines": true, "linewidth": 0, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -598,7 +598,7 @@ "lines": true, "linewidth": 0, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -671,7 +671,7 @@ "linewidth": 2, "links": [], "minSpan": 4, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -761,7 +761,7 @@ "linewidth": 2, "links": [], "minSpan": 4, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, diff --git a/var/lib/grafana/dashboards/Miscellaneous/Windows.json b/var/lib/grafana/dashboards/Miscellaneous/Windows.json index 5092a73e..cf363d48 100644 --- a/var/lib/grafana/dashboards/Miscellaneous/Windows.json +++ b/var/lib/grafana/dashboards/Miscellaneous/Windows.json @@ -53,7 +53,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -132,7 +132,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -212,7 +212,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -285,7 +285,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -347,7 +347,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -409,7 +409,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -471,7 +471,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -547,7 +547,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -609,7 +609,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, diff --git a/var/lib/grafana/dashboards/Miscellaneous/pfSense.json b/var/lib/grafana/dashboards/Miscellaneous/pfSense.json index 9a309f41..00223887 100644 --- a/var/lib/grafana/dashboards/Miscellaneous/pfSense.json +++ b/var/lib/grafana/dashboards/Miscellaneous/pfSense.json @@ -4,7 +4,7 @@ ] }, "description": "", - "editable": true, + "editable": false, "gnetId": null, "graphTooltip": 1, "id": null, @@ -63,7 +63,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -159,7 +159,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -271,7 +271,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -394,7 +394,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -510,7 +510,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -614,7 +614,7 @@ "lines": false, "linewidth": 0, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -749,7 +749,7 @@ "linewidth": 2, "links": [], "maxPerRow": 3, - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -871,7 +871,7 @@ "linewidth": 2, "links": [], "maxPerRow": 3, - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, diff --git a/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json b/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json index e237a664..0244ae31 100644 --- a/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json +++ b/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json @@ -174,7 +174,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -481,7 +481,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -666,7 +666,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -758,7 +758,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -861,7 +861,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -955,7 +955,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -1049,7 +1049,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -1147,7 +1147,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -1239,7 +1239,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -1331,7 +1331,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -1423,7 +1423,7 @@ "list": [] }, "time": { - "from": "now-3h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/SexiGraf/Netdata.json b/var/lib/grafana/dashboards/SexiGraf/Netdata.json index a9d3788a..9fdd013d 100644 --- a/var/lib/grafana/dashboards/SexiGraf/Netdata.json +++ b/var/lib/grafana/dashboards/SexiGraf/Netdata.json @@ -67,7 +67,7 @@ } ], "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "templating": { diff --git a/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Pull_Exec_Time.json b/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Pull_Exec_Time.json index a599ffee..f98626e3 100644 --- a/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Pull_Exec_Time.json +++ b/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Pull_Exec_Time.json @@ -74,6 +74,12 @@ "refId": "C", "target": "aliasByNode(vi.$ViServer.vm.exec.duration, 1, 2)", "textEditor": false + }, + { + "refCount": 0, + "refId": "D", + "target": "aliasByNode(veeam.vbr.$VbrServer.exec.duration, 2, 1)", + "textEditor": false } ], "thresholds": [], @@ -122,7 +128,8 @@ "tags": [ "VM", "VI", - "vSAN" + "vSAN", + "Veeam" ], "templating": { "list": [ @@ -149,6 +156,30 @@ "tagsQuery": "", "type": "query", "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": null, + "definition": "veeam.vbr.*", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "VbrServer", + "options": [], + "query": "veeam.vbr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false } ] }, diff --git a/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Web_Admin.json b/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Web_Admin.json index f8b42f57..939ef56f 100644 --- a/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Web_Admin.json +++ b/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Web_Admin.json @@ -33,7 +33,7 @@ } ], "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_Capacity_Planning.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_Capacity_Planning.json index 59c0a43e..afa1b6b9 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_Capacity_Planning.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_Capacity_Planning.json @@ -210,7 +210,7 @@ } ], "time": { - "from": "now-24h", + "from": "now-1d", "to": "now" }, "templating": { diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json index 3bd22909..20af7029 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json @@ -1352,6 +1352,190 @@ "alignLevel": null } }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 69, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dead" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "disabled" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "standby" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 18, + "links": [], + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "A", + "target": "sortBy(groupByNode(removeEmptySeries(vmw.$vcenter.$datacentre.$cluster.superstats.PathState.*), 6, 'total'), 'last', false)", + "textEditor": false + } + ], + "title": "MultipathState Count", + "type": "timeseries" + }, { "collapsed": false, "datasource": null, diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json index 1c5d4498..fa073418 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json @@ -1307,40 +1307,36 @@ } }, { - "aliasColors": { - "svmotion": "dark-orange", - "vmotion": "#052B51", - "xvmotion": "super-light-blue" - }, - "bars": true, + "aliasColors": {}, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "local", - "decimals": 0, + "decimals": 1, "editable": false, "error": false, - "fill": 4, + "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 6, - "w": 6, + "w": 12, "x": 0, "y": 31 }, "hiddenSeries": false, - "id": 4, + "id": 13, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": true, - "total": true, - "values": true + "show": false, + "total": false, + "values": false }, - "lines": false, - "linewidth": 2, + "lines": true, + "linewidth": 3, "links": [], "nullPointMode": "null", "options": { @@ -1351,36 +1347,39 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/vCPU/i", + "color": "#6D1F62" + } + ], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { + "hide": true, "refId": "A", - "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numVmotions), 'vmotion')", - "textEditor": false + "target": "vmw.$vcenter.$datacentre.$cluster.quickstats.pCPUs" }, { - "refCount": 0, + "hide": true, "refId": "B", - "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numSVMotions), 'svmotion')", - "textEditor": false + "target": "vmw.$vcenter.$datacentre.$cluster.quickstats.vCPUs" }, { - "refCount": 0, "refId": "C", - "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numXVMotions), 'xvmotion')", + "target": "alias(divideSeries(#B, #A), '$cluster.vCPU/pCPU')", "textEditor": false } ], "thresholds": [], "timeRegions": [], - "title": "vMotion count", + "title": "vCPU per pCPU ratio", "tooltip": { "shared": true, "sort": 2, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -1390,7 +1389,7 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "logBase": 1, "min": 0, "show": true @@ -1411,7 +1410,6 @@ "dashLength": 10, "dashes": false, "datasource": "local", - "decimals": 1, "editable": false, "error": false, "fill": 0, @@ -1420,11 +1418,11 @@ "gridPos": { "h": 6, "w": 6, - "x": 6, + "x": 12, "y": 31 }, "hiddenSeries": false, - "id": 13, + "id": 9, "legend": { "avg": false, "current": false, @@ -1435,7 +1433,7 @@ "values": false }, "lines": true, - "linewidth": 3, + "linewidth": 2, "links": [], "nullPointMode": "null", "options": { @@ -1446,35 +1444,20 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/vCPU/i", - "color": "#6D1F62" - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "hide": true, + "hide": false, "refId": "A", - "target": "vmw.$vcenter.$datacentre.$cluster.quickstats.pCPUs" - }, - { - "hide": true, - "refId": "B", - "target": "vmw.$vcenter.$datacentre.$cluster.quickstats.vCPUs" - }, - { - "refId": "C", - "target": "alias(divideSeries(#B, #A), '$cluster.vCPU/pCPU')", - "textEditor": false + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.esx.*.quickstats.distributedCpuFairness, 5)" } ], "thresholds": [], "timeRegions": [], - "title": "vCPU per pCPU ratio", + "title": "distributed CPU fairness", "tooltip": { "shared": true, "sort": 2, @@ -1488,12 +1471,13 @@ }, "yaxes": [ { + "$$hashKey": "object:1117", "format": "short", "logBase": 1, - "min": 0, "show": true }, { + "$$hashKey": "object:1118", "format": "short", "logBase": 1, "show": true @@ -1517,11 +1501,11 @@ "gridPos": { "h": 6, "w": 6, - "x": 12, + "x": 18, "y": 31 }, "hiddenSeries": false, - "id": 9, + "id": 18, "legend": { "avg": false, "current": false, @@ -1549,14 +1533,13 @@ "steppedLine": false, "targets": [ { - "hide": false, - "refId": "A", - "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.esx.*.quickstats.distributedCpuFairness, 5)" + "refId": "B", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.esx.*.quickstats.distributedMemoryFairness, 5)" } ], "thresholds": [], "timeRegions": [], - "title": "distributed CPU fairness", + "title": "distributed RAM fairness", "tooltip": { "shared": true, "sort": 2, @@ -1570,16 +1553,17 @@ }, "yaxes": [ { - "$$hashKey": "object:1117", + "$$hashKey": "object:1279", "format": "short", "logBase": 1, + "min": "0", "show": true }, { - "$$hashKey": "object:1118", + "$$hashKey": "object:1280", "format": "short", "logBase": 1, - "show": true + "show": false } ], "yaxis": { @@ -1587,34 +1571,39 @@ } }, { - "aliasColors": {}, - "bars": false, + "aliasColors": { + "svmotion": "dark-orange", + "vmotion": "#052B51", + "xvmotion": "super-light-blue" + }, + "bars": true, "dashLength": 10, "dashes": false, "datasource": "local", + "decimals": 0, "editable": false, "error": false, - "fill": 0, + "fill": 4, "fillGradient": 0, "grid": {}, "gridPos": { "h": 6, - "w": 6, - "x": 18, - "y": 31 + "w": 12, + "x": 0, + "y": 37 }, "hiddenSeries": false, - "id": 18, + "id": 4, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, - "total": false, - "values": false + "show": true, + "total": true, + "values": true }, - "lines": true, + "lines": false, "linewidth": 2, "links": [], "nullPointMode": "null", @@ -1628,21 +1617,34 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { + "refId": "A", + "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numVmotions), 'vmotion')", + "textEditor": false + }, + { + "refCount": 0, "refId": "B", - "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.esx.*.quickstats.distributedMemoryFairness, 5)" + "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numSVMotions), 'svmotion')", + "textEditor": false + }, + { + "refCount": 0, + "refId": "C", + "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numXVMotions), 'xvmotion')", + "textEditor": false } ], "thresholds": [], "timeRegions": [], - "title": "distributed RAM fairness", + "title": "vMotion count", "tooltip": { "shared": true, "sort": 2, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -1652,23 +1654,205 @@ }, "yaxes": [ { - "$$hashKey": "object:1279", - "format": "short", + "format": "none", "logBase": 1, - "min": "0", + "min": 0, "show": true }, { - "$$hashKey": "object:1280", "format": "short", "logBase": 1, - "show": false + "show": true } ], "yaxis": { "align": false } }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 69, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dead" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "disabled" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "standby" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 20, + "links": [], + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "A", + "target": "sortBy(groupByNode(removeEmptySeries(vmw.$vcenter.$datacentre.$cluster.superstats.PathState.*), 6, 'total'), 'last', false)", + "textEditor": false + } + ], + "title": "MultipathState Count", + "type": "timeseries" + }, { "collapsed": false, "datasource": "local", @@ -1676,7 +1860,7 @@ "h": 1, "w": 24, "x": 0, - "y": 37 + "y": 43 }, "id": 17, "panels": [], @@ -1698,7 +1882,7 @@ "h": 6, "w": 12, "x": 0, - "y": 38 + "y": 44 }, "hiddenSeries": false, "id": 11, @@ -1795,7 +1979,7 @@ "h": 6, "w": 12, "x": 12, - "y": 38 + "y": 44 }, "hiddenSeries": false, "id": 12, diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Hardware_Sensor.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Hardware_Sensor.json index b58ebbf7..409ac8b4 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Hardware_Sensor.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Hardware_Sensor.json @@ -101,7 +101,7 @@ "label": "", "logBase": 1, "max": null, - "min": 0, + "min": null, "show": true }, { @@ -109,7 +109,7 @@ "format": "none", "logBase": 1, "max": null, - "min": "0", + "min": null, "show": true } ], diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Multi_ESX_LiteStats.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Multi_ESX_LiteStats.json index f2647232..7c2c16f0 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Multi_ESX_LiteStats.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Multi_ESX_LiteStats.json @@ -497,7 +497,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_DRS_Score.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_DRS_Score.json new file mode 100644 index 00000000..b40f656c --- /dev/null +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_DRS_Score.json @@ -0,0 +1,389 @@ +{ + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "iteration": 1696154080221, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "title": "$cluster", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "VM", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "0_20" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "21_40" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "41_60" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "61_80" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "81_100" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ClusterDrsScore" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + }, + { + "id": "max", + "value": 100 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.axisLabel", + "value": "" + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 19, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.4.4", + "repeat": "cluster", + "repeatDirection": "h", + "targets": [ + { + "datasource": null, + "hide": false, + "refCount": 0, + "refId": "A", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.0_20, 6)", + "textEditor": false + }, + { + "datasource": null, + "hide": false, + "refCount": 0, + "refId": "B", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.21_40, 6)", + "textEditor": false + }, + { + "datasource": null, + "hide": false, + "refCount": 0, + "refId": "C", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.41_60, 6)", + "textEditor": false + }, + { + "datasource": null, + "hide": false, + "refCount": 0, + "refId": "D", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.61_80, 6)", + "textEditor": false + }, + { + "datasource": null, + "hide": false, + "refCount": 0, + "refId": "E", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.81_100, 6)", + "textEditor": false + }, + { + "datasource": null, + "hide": false, + "refCount": 0, + "refId": "F", + "target": "alias(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.DrsScore, 'ClusterDrsScore')", + "textEditor": false + } + ], + "title": "DrsScoreBucket $cluster", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 35, + "style": "light", + "tags": [], + "templating": { + "list": [ + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "multiFormat": "glob", + "name": "vcenter", + "options": [], + "query": "vmw.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "multiFormat": "glob", + "name": "datacentre", + "options": [], + "query": "vmw.$vcenter.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "glob", + "allValue": "*", + "current": {}, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "multiFormat": "glob", + "name": "cluster", + "options": [], + "query": "vmw.$vcenter.$datacentre.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5m", + "15m", + "1h", + "2h", + "3h", + "6h", + "12h", + "24h" + ], + "status": "Stable", + "time_options": [ + "5m", + "15m", + "30m", + "1h", + "2h", + "1d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "VMware Cluster Multi DRS Score", + "uid": "whn1iUZIk", + "version": 11, + "weekStart": "" + } \ No newline at end of file diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_vMotion.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_vMotion.json index 348e9157..b8d42d8f 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_vMotion.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_vMotion.json @@ -152,7 +152,7 @@ } ], "time": { - "from": "now-1d", + "from": "now-1h", "to": "now" }, "templating": { diff --git a/var/lib/grafana/dashboards/VMware_Datastore/VMware_All_Datastore_Usage_Distribution.json b/var/lib/grafana/dashboards/VMware_Datastore/VMware_All_Datastore_Usage_Distribution.json index 8a70e5ed..d1ddf616 100644 --- a/var/lib/grafana/dashboards/VMware_Datastore/VMware_All_Datastore_Usage_Distribution.json +++ b/var/lib/grafana/dashboards/VMware_Datastore/VMware_All_Datastore_Usage_Distribution.json @@ -52,7 +52,7 @@ "lines": true, "linewidth": 0, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, diff --git a/var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Gauge.json b/var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Gauge.json new file mode 100644 index 00000000..635dc93e --- /dev/null +++ b/var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Gauge.json @@ -0,0 +1,251 @@ +{ + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "iteration": 1587995684562, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "repeat": null, + "title": "", + "type": "row" + }, + { + "datasource": null, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 8, + "maxPerRow": 6, + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "max": 104, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 255, 0)", + "value": null + }, + { + "color": "rgb(255, 239, 0)", + "value": 70 + }, + { + "color": "rgb(255, 115, 0)", + "value": 80 + }, + { + "color": "rgb(255, 0, 29)", + "value": 90 + }, + { + "color": "rgb(204, 0, 255)", + "value": 99 + } + ] + }, + "unit": "percent" + }, + "overrides": [], + "values": false + }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": false + }, + "pluginVersion": "6.7.1", + "repeat": "datastore", + "repeatDirection": "h", + "targets": [ + { + "refCount": 0, + "refId": "B", + "target": "alias(removeEmptySeries(vmw.$vcenter.$datacentre.$cluster.datastore.$datastore.summary.usagePct), 'usagePct')", + "textEditor": false + } + ], + "timeFrom": null, + "timeShift": null, + "title": "$datastore", + "type": "bargauge" + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "light", + "tags": [ + "HDD" + ], + "templating": { + "list": [ + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": "local", + "definition": "vmw.*", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "vcenter", + "options": [], + "query": "vmw.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": "local", + "definition": "vmw.$vcenter.*", + "hide": 0, + "includeAll": true, + "index": -1, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "datacentre", + "options": [], + "query": "vmw.$vcenter.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": "local", + "definition": "vmw.$vcenter.$datacentre.*", + "hide": 0, + "includeAll": true, + "index": -1, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "cluster", + "options": [], + "query": "vmw.$vcenter.$datacentre.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": "local", + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "datastore", + "options": [], + "query": "vmw.$vcenter.$datacentre.$cluster.datastore.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "15m", + "1h", + "2h", + "3h", + "6h", + "12h", + "24h", + "2d", + "3d", + "7d", + "15d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "VMware Multi Datastore Gauge", + "uid": "6e3fb4e8", + "variables": { + "list": [] + }, + "version": 14 +} \ No newline at end of file diff --git a/var/lib/grafana/dashboards/VMware_Inventory/VMware_Datastore_Inventory.json b/var/lib/grafana/dashboards/VMware_Inventory/VMware_Datastore_Inventory.json index 33081726..13c031cd 100644 --- a/var/lib/grafana/dashboards/VMware_Inventory/VMware_Datastore_Inventory.json +++ b/var/lib/grafana/dashboards/VMware_Inventory/VMware_Datastore_Inventory.json @@ -77,7 +77,7 @@ "list": [] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_Inventory/VMware_ESX_Inventory.json b/var/lib/grafana/dashboards/VMware_Inventory/VMware_ESX_Inventory.json index 8cca5577..fc727e01 100644 --- a/var/lib/grafana/dashboards/VMware_Inventory/VMware_ESX_Inventory.json +++ b/var/lib/grafana/dashboards/VMware_Inventory/VMware_ESX_Inventory.json @@ -77,7 +77,7 @@ "list": [] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json new file mode 100644 index 00000000..7969e1b9 --- /dev/null +++ b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json @@ -0,0 +1,91 @@ +{ + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "ViVbrCsv", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 32, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "showHeader": true + }, + "pluginVersion": "7.5.11", + "targets": [ + { + "decimalSeparator": ".", + "delimiter": ",", + "experimental": { + "regex": true + }, + "header": true, + "ignoreUnknown": false, + "refId": "A", + "schema": [ + { + "name": "", + "type": "string" + } + ], + "skipRows": 0 + } + ], + "title": " ", + "type": "table" + } + ], + "schemaVersion": 27, + "style": "dark", + "tags": [ + "VI", + "VM", + "VBR" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true + }, + "timezone": "", + "title": "VMware VBR Inventory", + "uid": "iQlLNW57c", + "version": 8 +} \ No newline at end of file diff --git a/var/lib/grafana/dashboards/VMware_Inventory/VMware_VM_Inventory.json b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VM_Inventory.json index 9e167f33..5cc5456f 100644 --- a/var/lib/grafana/dashboards/VMware_Inventory/VMware_VM_Inventory.json +++ b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VM_Inventory.json @@ -77,7 +77,7 @@ "list": [] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json index b03256ff..593b1b1c 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json @@ -614,6 +614,81 @@ "show": true } ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "id": 14, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$cluster.vm.$vm.fatstats.diskIOPS, '$N')), 5)", + "textEditor": true + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } ], "repeat": null, @@ -1012,7 +1087,7 @@ "error": false, "fill": 0, "grid": {}, - "id": 14, + "id": 15, "legend": { "avg": false, "current": false, diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json index 07d12a13..be25962f 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json @@ -614,6 +614,156 @@ "show": true } ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "id": 14, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.vm.$vm.fatstats.diskIOPS, 5)", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "id": 15, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "sortByMaxima(aliasByNode(veeam.vi.$vcenter.$cluster.vm.$vm.restorePointsCount, 5))", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Veeam Backup Restore Points", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } ], "repeat": null, diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json index cc01e4bd..44390e10 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json @@ -614,6 +614,81 @@ "show": true } ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "id": 14, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(sortByMaxima(highest$view(esx.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskIOPS, '$N')), 5)", + "textEditor": true + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } ], "repeat": null, diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json index c203877a..35c2e460 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json @@ -614,6 +614,81 @@ "show": true } ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "id": 14, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(sortByMaxima(highest$view(esx.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskIOPS, '$N')), 5)", + "textEditor": true + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } ], "repeat": null, diff --git a/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json b/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json index 1ebabb24..f2140d8a 100644 --- a/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json +++ b/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json @@ -504,14 +504,14 @@ "list": [ { "allFormat": "wildcard", - "allValue": null, + "allValue": "*", "current": {}, "datasource": null, "definition": "vi.*", "description": null, "error": null, "hide": 0, - "includeAll": false, + "includeAll": true, "label": null, "multi": true, "multiFormat": "glob", diff --git a/var/lib/grafana/dashboards/VMware_vCenter/VMware_BroStats.json b/var/lib/grafana/dashboards/VMware_vCenter/VMware_BroStats.json index 72a2af92..021a03f5 100644 --- a/var/lib/grafana/dashboards/VMware_vCenter/VMware_BroStats.json +++ b/var/lib/grafana/dashboards/VMware_vCenter/VMware_BroStats.json @@ -856,7 +856,7 @@ "list": [] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json b/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json new file mode 100644 index 00000000..30519dad --- /dev/null +++ b/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json @@ -0,0 +1,469 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "iteration": 1617871211625, + "links": [], + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 61, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": null, + "hide": false, + "refId": "A", + "target": "groupByNode(vi.$vcenter.vi.version.vm.$datacentre.$cluster.vmtools.*, 8, 'sum')" + } + ], + "title": "VMTools version", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 61, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": null, + "hide": false, + "refId": "A", + "target": "groupByNode(vi.$vcenter.vi.version.vm.$datacentre.$cluster.guest.*, 8, 'sum')" + } + ], + "title": "GuestId version", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 61, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": null, + "hide": false, + "refId": "A", + "target": "groupByNode(vi.$vcenter.vi.version.vm.$datacentre.$cluster.vhw.*, 8, 'sum')" + } + ], + "title": "vHardware version", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": null, + "hide": false, + "refCount": 0, + "refId": "B", + "target": "groupByNode(removeEmptySeries(vi.$vcenter.vi.version.esx.$datacentre.$cluster.build.*), 8, 'sum')" + } + ], + "title": "ESX version", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 27, + "style": "light", + "tags": [ + "VI" + ], + "templating": { + "list": [ + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": null, + "definition": "vi.*", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "vcenter", + "options": [], + "query": "vi.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": null, + "definition": "vi.$vcenter.vi.version.esx.*", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "datacentre", + "options": [], + "query": "vi.$vcenter.vi.version.esx.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": null, + "definition": "vi.$vcenter.vi.version.esx.$datacentre.*", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "cluster", + "options": [], + "query": "vi.$vcenter.vi.version.esx.$datacentre.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1d", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "1d", + "2d", + "3d", + "4d", + "5d", + "6d", + "7d", + "15d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "VMware Evo Version", + "uid": "v5Re8clMz", + "version": 30 +} \ No newline at end of file diff --git a/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Resync.json b/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Resync.json index 40a01d4d..51b0e5b4 100644 --- a/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Resync.json +++ b/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Resync.json @@ -394,7 +394,7 @@ } ], "maxPerRow": 1, - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, diff --git a/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Space_Usage_Report.json b/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Space_Usage_Report.json index 31f53a19..0ac00ad1 100644 --- a/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Space_Usage_Report.json +++ b/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Space_Usage_Report.json @@ -332,7 +332,7 @@ ] }, "time": { - "from": "now-3h", + "from": "now-1d", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_vSAN/VMware_vSAN_SMART_Stats.json b/var/lib/grafana/dashboards/VMware_vSAN/VMware_vSAN_SMART_Stats.json index 77c82565..62127b6c 100644 --- a/var/lib/grafana/dashboards/VMware_vSAN/VMware_vSAN_SMART_Stats.json +++ b/var/lib/grafana/dashboards/VMware_vSAN/VMware_vSAN_SMART_Stats.json @@ -3,7 +3,7 @@ "list": [ ] }, - "editable": true, + "editable": false, "gnetId": null, "graphTooltip": 1, "id": null, diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json new file mode 100644 index 00000000..ba3d42bd --- /dev/null +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json @@ -0,0 +1,150 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "iteration": 1683841492842, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "fillOpacity": 100, + "lineWidth": 1 + }, + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "Success" + }, + "1": { + "color": "yellow", + "index": 1, + "text": "Warning" + }, + "2": { + "color": "red", + "index": 2, + "text": "Failed" + }, + "3": { + "color": "#808080", + "index": 3, + "text": "unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 31, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "hidden", + "placement": "bottom" + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "B", + "target": "sortByName(aliasByNode(removeEmptySeries(veeam.vbr.$vbrserver.job.$job.result), 4), false)" + } + ], + "title": "Job Result", + "type": "status-history" + } + ], + "refresh": false, + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vbrserver", + "options": [], + "query": "veeam.vbr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.$vbrserver.job.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": "veeam.vbr.$vbrserver.job.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + } + ] + }, + "time": { + "from": "now-1d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Veeam Job Result", + "uid": "oAhiTQyVk", + "version": 14, + "weekStart": "" +} \ No newline at end of file diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json new file mode 100644 index 00000000..d0fe39bc --- /dev/null +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json @@ -0,0 +1,151 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "iteration": 1683841799856, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "running" + }, + "1": { + "color": "yellow", + "index": 1, + "text": "inactive" + }, + "2": { + "color": "red", + "index": 2, + "text": "disabled" + }, + "3": { + "color": "#808080", + "index": 3, + "text": "unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 31, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "mergeValues": true, + "rowHeight": 0.8, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": "local", + "refId": "A", + "target": "sortByName(aliasByNode(removeEmptySeries(veeam.vbr.$vbrserver.job.$job.status), 4), false)", + "textEditor": false + } + ], + "title": "Job Status", + "transformations": [], + "type": "state-timeline" + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vbrserver", + "options": [], + "query": "veeam.vbr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.$vbrserver.job.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": "veeam.vbr.$vbrserver.job.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + } + ] + }, + "time": { + "from": "now-1d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Veeam Job Status", + "uid": "vo7u77UVk", + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json new file mode 100644 index 00000000..312087a3 --- /dev/null +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json @@ -0,0 +1,229 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "id": null, + "iteration": 1683841959668, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": { + "capacity": "#BF1B00", + "uncommitted": "#EAB839", + "used": "#0A437C" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": false, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 4, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "repo", + "repeatDirection": "h", + "seriesOverrides": [ + { + "$$hashKey": "object:416", + "alias": "capacity", + "fill": 0, + "stack": false + }, + { + "$$hashKey": "object:708", + "alias": "RealUsedPct", + "legend": true, + "lines": false, + "stack": false, + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "A", + "target": "alias(veeam.vbr.$vbrserver.repo.$repo.usedSpaceGB, 'used')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "B", + "target": "alias(veeam.vbr.$vbrserver.repo.$repo.freeGB, 'free')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "C", + "target": "alias(veeam.vbr.$vbrserver.repo.$repo.capacityGB, 'capacity')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "D", + "target": "aliasByNode(veeam.vbr.$vbrserver.repo.$repo.RealUsedPct, 5)", + "textEditor": false + } + ], + "thresholds": [], + "timeRegions": [], + "title": "$repo usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:410", + "format": "decgbytes", + "logBase": 1, + "min": 0, + "show": true + }, + { + "$$hashKey": "object:411", + "format": "percent", + "logBase": 1, + "max": "100", + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 35, + "style": "light", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vbrserver", + "options": [], + "query": "veeam.vbr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.$vbrserver.repo.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "repo", + "options": [], + "query": "veeam.vbr.$vbrserver.repo.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "datasource": "local" + } + ] + }, + "time": { + "from": "now-1d", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "15m", + "1h", + "2h", + "3h", + "6h", + "12h", + "24h", + "2d", + "3d", + "7d", + "15d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Veeam Multi Repository Usage", + "uid": "ofgN-MyVz", + "version": 11, + "weekStart": "" +} \ No newline at end of file diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json new file mode 100644 index 00000000..58330fac --- /dev/null +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json @@ -0,0 +1,229 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "id": null, + "iteration": 1683841959668, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": { + "capacity": "#BF1B00", + "uncommitted": "#EAB839", + "used": "#0A437C" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": false, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 4, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "sobr", + "repeatDirection": "h", + "seriesOverrides": [ + { + "$$hashKey": "object:416", + "alias": "capacity", + "fill": 0, + "stack": false + }, + { + "$$hashKey": "object:708", + "alias": "RealUsedPct", + "legend": true, + "lines": false, + "stack": false, + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "A", + "target": "alias(veeam.vbr.$vbrserver.sobr.$sobr.usedSpaceGB, 'used')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "B", + "target": "alias(veeam.vbr.$vbrserver.sobr.$sobr.freeGB, 'free')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "C", + "target": "alias(veeam.vbr.$vbrserver.sobr.$sobr.capacityGB, 'capacity')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "D", + "target": "aliasByNode(veeam.vbr.$vbrserver.sobr.$sobr.RealUsedPct, 5)", + "textEditor": false + } + ], + "thresholds": [], + "timeRegions": [], + "title": "$sobr usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:410", + "format": "decgbytes", + "logBase": 1, + "min": 0, + "show": true + }, + { + "$$hashKey": "object:411", + "format": "percent", + "logBase": 1, + "max": "100", + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 35, + "style": "light", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vbrserver", + "options": [], + "query": "veeam.vbr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.$vbrserver.sobr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "sobr", + "options": [], + "query": "veeam.vbr.$vbrserver.sobr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "datasource": "local" + } + ] + }, + "time": { + "from": "now-1d", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "15m", + "1h", + "2h", + "3h", + "6h", + "12h", + "24h", + "2d", + "3d", + "7d", + "15d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Veeam Multi SOBR Usage", + "uid": "ofgN-NyVz", + "version": 11, + "weekStart": "" +} \ No newline at end of file diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json new file mode 100644 index 00000000..d62a20f2 --- /dev/null +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json @@ -0,0 +1,259 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "iteration": 1683841856790, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": "local", + "refId": "A", + "target": "sortByTotal(aliasByNode(removeEmptySeries(veeam.vbr.$vbrserver.job.$job.objectRestorePoints), 4))" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Restore Points Per Job", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:37", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:38", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": "local", + "refId": "A", + "target": "groupByNode(removeEmptySeries(veeam.vi.$vcenter.$cluster.objectRestorePoints), 3, 'total')" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Restore Points Per Cluster", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:37", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:38", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vbrserver", + "options": [], + "query": "veeam.vbr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.$vbrserver.job.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": "veeam.vbr.$vbrserver.job.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vi.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vcenter", + "options": [], + "query": "veeam.vi.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vi.$vcenter.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "cluster", + "options": [], + "query": "veeam.vi.$vcenter.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + } + ] + }, + "time": { + "from": "now-1d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Veeam Restore Point", + "uid": "ST65OiPVk", + "version": 23, + "weekStart": "" +} \ No newline at end of file diff --git a/var/www/admin/credstore.php b/var/www/admin/credstore.php index 997f08fe..32b70959 100644 --- a/var/www/admin/credstore.php +++ b/var/www/admin/credstore.php @@ -6,14 +6,14 @@ ?>

-

Credential Store Notes

+

vSphere Credential Store Notes

    -
  • The credential store is used to store credential that will be used for vCenter/ESX query, it use vSphere SDK Credential Store Library
    • +
  • The vSphere credential store is used to store credential that will be used for vCenter/ESX query, it uses vSphere SDK Credential Store Library
  • Removing a vCenter/ESX from the credential store will NOT delete any collected metrics.
  • Please refer to the project website and documentation for more information.
-

SexiGraf Credential Store

+

SexiGraf vSphere Credential Store

diff --git a/var/www/admin/header.php b/var/www/admin/header.php index eaa9718f..cd27aa23 100644 --- a/var/www/admin/header.php +++ b/var/www/admin/header.php @@ -36,9 +36,10 @@
  • Summary
  • vSphere Credential Store
    • +
  • Veeam Credential Store
  • Package Updater
  • House Cleaner
    • -
  • Refresh Inventory
    • +
  • Inventory Refresh & History
  • Log Viewer
  • Export / Import
    • diff --git a/var/www/admin/helper.php b/var/www/admin/helper.php index f31863a9..3b7f3d23 100644 --- a/var/www/admin/helper.php +++ b/var/www/admin/helper.php @@ -8,6 +8,11 @@ function isViEnabled($inputvcenter) { return (file_exists($crontabPath . "vi_" . str_replace(".", "_", $inputvcenter))); } +function isVbrEnabled($inputvbrserver) { + global $crontabPath; + return (file_exists($crontabPath . "vbr_" . str_replace(".", "_", $inputvbrserver))); +} + function isVsanEnabled($inputvcenter) { global $crontabPath; return (file_exists($crontabPath . "vsan_" . str_replace(".", "_", $inputvcenter))); @@ -22,10 +27,14 @@ function enableVi($inputvcenter) { shell_exec("sudo /bin/bash /var/www/scripts/a function enableVsan($inputvcenter) { shell_exec("sudo /bin/bash /var/www/scripts/addVsanCrontab.sh " . $inputvcenter); } +function enableVbr($inputvbrserver) { shell_exec("sudo /bin/bash /var/www/scripts/addVbrCrontab.sh " . $inputvbrserver); } + function disableVi($inputvcenter) { shell_exec("sudo /bin/bash /var/www/scripts/removeViCrontab.sh " . $inputvcenter); } function disableVsan($inputvcenter) { shell_exec("sudo /bin/bash /var/www/scripts/removeVsanCrontab.sh " . $inputvcenter); } +function disableVbr($inputvbrserver) { shell_exec("sudo /bin/bash /var/www/scripts/removeVbrCrontab.sh " . $inputvbrserver); } + function enableAutopurge($nbPurgeDays) { file_put_contents('./graphite_autopurge', $nbPurgeDays); shell_exec("sudo /bin/bash /var/www/scripts/addAutopurgeCrontab.sh " . $nbPurgeDays); diff --git a/var/www/admin/index.php b/var/www/admin/index.php index 4619f44d..e14bc36c 100644 --- a/var/www/admin/index.php +++ b/var/www/admin/index.php @@ -13,7 +13,7 @@
    -
    Credential Store
    +
    vSphere Credential Store
    @@ -65,7 +65,7 @@ -
    +
    @@ -75,12 +75,12 @@
    -
    Refresh Inventory
    +
    Inventory Refresh & History
    - The VI Offline Inventory is automatically schedule to be updated every hours. If you want to force a refresh, you can use this section to perform update. + The VMware and Veeam inventories are automatically updated every hours. If you need to force a refresh sooner, you can use this section to force the update.
    @@ -127,6 +127,28 @@ +
    +
    + +
    +
    +
    +
    + +
    +
    +
    Veeam Credential Store
    +
    +
    +
    +
    + In this section, you'll be able to manage your Veeam Backup & Replication Credential Store. This store is used to set up your Backup Server information to allow SexiGraf query what's needed. +
    +
    +
    +     +
    +
    diff --git a/var/www/admin/purge.php b/var/www/admin/purge.php index ca98dbc4..81bcee37 100644 --- a/var/www/admin/purge.php +++ b/var/www/admin/purge.php @@ -10,7 +10,8 @@
    • This page can be used to purge old and/or unwanted whisper data objects as well as vCenter session files.
    • Beware as this operation cannot be undone, so there is a risk of DATA LOSS if you don't know what you're doing.
      • -
    • Autopurge will automatically remove all files that are not updated since selected days (i.e. orphaned vms).
      • +
    • Autopurge will automatically remove all files not updated since selected days (i.e. deleted vms).
      • +
    • You can check the WhisperAutoPurge log in the Log Viewer panel to follow the process.
    • Please refer to the project website and documentation for more information.
    @@ -76,7 +77,7 @@
    - ' . php_file_tree_dir_v2("/mnt/wfs/whisper", 2) . ' + ' . php_file_tree_dir_v2("/mnt/wfs/whisper", 3) . '
    ' . php_file_tree("/tmp", "dat") . ' @@ -109,7 +110,7 @@
    - Force purge items older than days
    + Force purge deleted items older than days
    diff --git a/var/www/admin/refresh-inventory.php b/var/www/admin/refresh-inventory.php index 7bab6ebf..5c82520d 100644 --- a/var/www/admin/refresh-inventory.php +++ b/var/www/admin/refresh-inventory.php @@ -1,6 +1,6 @@ @@ -8,7 +8,7 @@

    Refresh VI Offline Inventory Notes

      -
    • The Static Offline Inventory is automatically schedule to be updated every 1 hour.
      • +
    • The Static VI Offline Inventory is automatically updated every hour.
    • If you want to force a refresh, you can use this section to perform update
    @@ -32,7 +32,7 @@

    Warning: Confirmation needed!

    - You are about to force inventory update. It should be use only for DEBUG purpose as it is already scheduled to run hourly.
    The process itself can take a few seconds (or minutes depending on your platform size). Are you sure about this? We mean, really sure?
    + You are about to force VI inventory update. It should be use only for DEBUG purpose as it is already scheduled to run hourly.
    The process itself can take a few seconds (or minutes depending on your platform size). Are you sure about this? We mean, really sure?

    '; @@ -50,11 +50,25 @@ } } else { echo '
    -

    +

    '; } ?> + +'.$filename.''; + } + } +?> +

    Inventory History:

    +
    + diff --git a/var/www/admin/showlog.php b/var/www/admin/showlog.php index 49cd2ad9..de43ca67 100644 --- a/var/www/admin/showlog.php +++ b/var/www/admin/showlog.php @@ -227,6 +227,7 @@ function updateLog() {
  • Summary
  • vSphere Credential Store
    • +
  • Veeam Credential Store
  • Package Updater
  • Stats Remover
  • Refresh Inventory
    • @@ -234,7 +235,7 @@ function updateLog() {
  • Export / Import
    • - + @@ -255,8 +256,10 @@ function updateLog() { $tail = new PHPTail(array( "VI Offline Inventory" => "/var/log/sexigraf/ViOfflineInventory.log", - "vSAN Statistics" => "/var/log/sexigraf/VsanDisksPullStatistics.log", "VI Statistics" => "/var/log/sexigraf/ViPullStatistics.log", + "VBR Statistics" => "/var/log/sexigraf/VbrPullStatistics.log", + "xMotion Garbage Collector" => "/var/log/sexigraf/xMotionGarbageCollector.log", + "Whisper Auto Purge" => "/var/log/sexigraf/WhisperAutoPurge.log", )); /** diff --git a/var/www/admin/vbrcredstore.php b/var/www/admin/vbrcredstore.php new file mode 100644 index 00000000..e7074032 --- /dev/null +++ b/var/www/admin/vbrcredstore.php @@ -0,0 +1,168 @@ + +

    +
    +

    Veeam Credential Store Notes

    +
      +
    • The Veeam credential store is used to store credential that will be used for Veeam Backup Server query, it uses the RESTful API for VBR (v11+)
      • +
    • Removing a VBR Server from the credential store will NOT delete any collected metrics.
      • +
    • Please refer to the project website and documentation for more information.
      • +
    +
    +

    SexiGraf Veeam Credential Store

    +
    vCenter/ESX address
    + + + + + + + + + + + + '; + if (isVbrEnabled($lineObjects[0])) { + echo ' '; + } else { + echo ' '; + } + echo ' + +'; + } + } +?> + + + + + + + + +
    VBR Server addressUsernamePasswordEnabled 
    ' . $lineObjects[0] . "" . $lineObjects[1] . '***********
    + + +
    + +
      '; + if (isVbrEnabled($lineObjects[0])) { + echo '
      • '; + } else { + echo '
      • '; + } + echo ' +
      • +
    +
    +
     *
    + + 0) { + $errorHappened = true; + $errorMessage = "VBR IP or FQDN is already in credential store, duplicate entry is not supported."; + } elseif (preg_match("/^([a-zA-Z0-9-_.]*)\\\\?([a-zA-Z0-9-_.]+)$|^([a-zA-Z0-9-_.]*)$|^([a-zA-Z0-9-_.]+)@([a-zA-Z0-9-_.]*)$/", $_POST["input-username"]) == 0) { + $errorHappened = true; + $errorMessage = "Wrong username format, supported format are DOMAIN\USERNAME, USERNAME, USERNAME@DOMAIN.TLD"; + } else { + exec("/usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/VbrConnect.ps1 -server " . escapeshellcmd($_POST["input-vbr"]) . " -username " . escapeshellcmd($_POST["input-username"]) . " -password " . escapeshellcmd($_POST["input-password"]), $null, $return_var); + if ($return_var) { + $errorHappened = true; + $errorMessage = "Wrong username/password, AccessDenied or no answer at TCP:9419"; + } + } + + if ($errorHappened) { + echo ' '; + echo ''; + } else { + echo ' '; + echo ''; + } + break; + case "delete-vcentry": + echo '
    +

    + Warning: + Confirmation needed!

    + You are about to delete entry from Veeam Credential Store for ' . $_POST["input-vbr"] . '. Are you sure about this? We mean, really sure?
    +
    + + +

    +
    '; + echo '
    '; + break; + case "delete-vcentry-confirmed": + disableVbr($_POST["input-vbr"]); + echo ' '; + echo ''; + break; + case "enable-vbr": + enableVbr($_POST["input-vbr"]); + echo ' + '; + break; + case "disable-vbr": + disableVbr($_POST["input-vbr"]); + echo ' + '; + break; + } + } +?> +
    + + + diff --git a/var/www/scripts/addVbrCrontab.sh b/var/www/scripts/addVbrCrontab.sh new file mode 100644 index 00000000..3fa1367b --- /dev/null +++ b/var/www/scripts/addVbrCrontab.sh @@ -0,0 +1,5 @@ +#!/bin/bash +sessionFile="/tmp/vbr_$(sed s/\\./_/g <<<$1).key" +crontabFile="/etc/cron.d/vbr_$(sed s/\\./_/g <<<$1)" +echo "*/5 * * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/VbrPullStatistics.ps1 -credstore /mnt/wfs/inventory/vbrpscredentials.xml -server $1 -sessionfile $sessionFile >/dev/null 2>&1" >> $crontabFile +service cron reload \ No newline at end of file diff --git a/var/www/scripts/exportSexiGrafBundle.sh b/var/www/scripts/exportSexiGrafBundle.sh index f0a13cf9..3782f7a2 100644 --- a/var/www/scripts/exportSexiGrafBundle.sh +++ b/var/www/scripts/exportSexiGrafBundle.sh @@ -25,6 +25,7 @@ if [ -d "/mnt/wfs/whisper" ]; then # Retrieve cron entries cp /etc/cron.d/vi_* /mnt/wfs/sexigraf-dump/conf/cron.d/ cp /etc/cron.d/vsan_* /mnt/wfs/sexigraf-dump/conf/cron.d/ + cp /etc/cron.d/vbr_* /mnt/wfs/sexigraf-dump/conf/cron.d/ # Retrieve version file cp /etc/sexigraf_version /mnt/wfs/sexigraf-dump/ @@ -33,24 +34,28 @@ if [ -d "/mnt/wfs/whisper" ]; then cp /mnt/wfs/inventory/*.csv /mnt/wfs/sexigraf-dump/ # Retrieve credential store items - if [ -a "/var/www/.vmware/credstore/vicredentials.xml" ]; then - for creditem in $(/usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml list | egrep -v "Server|^$" | sed "s/[[:space:]]\+/;/") - do - vcenter=$(echo $creditem | cut -d ";" -f 1) - username=$(echo $creditem | cut -d ";" -f 2) - password="/usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml get --server $vcenter --username '$username'" - password=$(eval "$password" | cut -c 11-) - echo "$vcenter;$username;$password" >> /mnt/wfs/sexigraf-dump/conf/vicredentials.conf - done - # File encoding - openssl des3 -salt -in /mnt/wfs/sexigraf-dump/conf/vicredentials.conf -out /mnt/wfs/sexigraf-dump/conf/vicredentials.conf.ss -pass pass:sexigraf - rm -f /mnt/wfs/sexigraf-dump/conf/vicredentials.conf - fi + # if [ -a "/var/www/.vmware/credstore/vicredentials.xml" ]; then + # for creditem in $(/usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml list | egrep -v "Server|^$" | sed "s/[[:space:]]\+/;/") + # do + # vcenter=$(echo $creditem | cut -d ";" -f 1) + # username=$(echo $creditem | cut -d ";" -f 2) + # password="/usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml get --server $vcenter --username '$username'" + # password=$(eval "$password" | cut -c 11-) + # echo "$vcenter;$username;$password" >> /mnt/wfs/sexigraf-dump/conf/vicredentials.conf + # done + # # File encoding + # openssl des3 -salt -in /mnt/wfs/sexigraf-dump/conf/vicredentials.conf -out /mnt/wfs/sexigraf-dump/conf/vicredentials.conf.ss -pass pass:sexigraf + # rm -f /mnt/wfs/sexigraf-dump/conf/vicredentials.conf + # fi if [ -a "/mnt/wfs/inventory/vipscredentials.xml" ]; then cp /mnt/wfs/inventory/vipscredentials.xml /mnt/wfs/sexigraf-dump/conf/vipscredentials.xml fi + if [ -a "/mnt/wfs/inventory/vbrpscredentials.xml" ]; then + cp /mnt/wfs/inventory/vbrpscredentials.xml /mnt/wfs/sexigraf-dump/conf/vbrpscredentials.xml + fi + # create ISO file from export folder /usr/bin/genisoimage --allow-limited-size -udf -f -J -joliet-long -r -U -iso-level 4 -o /mnt/wfs/sexigraf-dump.iso /mnt/wfs/sexigraf-dump ln -s /mnt/wfs/sexigraf-dump.iso /var/www/admin/sexigraf-dump.iso diff --git a/var/www/scripts/importSexiGrafBundle.sh b/var/www/scripts/importSexiGrafBundle.sh index dfa17958..fc28961a 100644 --- a/var/www/scripts/importSexiGrafBundle.sh +++ b/var/www/scripts/importSexiGrafBundle.sh @@ -38,26 +38,31 @@ if [ -d "/mnt/wfs/whisper" ]; then # chmod 644 /var/www/admin/offline-vminventory.html # Import credential store items - if [ -a "/var/www/.vmware/credstore/vicredentials.xml" ]; then - openssl des3 -d -salt -in /media/cdrom/conf/vicredentials.conf.ss -out /tmp/vicredentials.conf -pass pass:sexigraf -md md5 - # /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/CredstoreAdmin.ps1 -createstore -credstore /mnt/wfs/inventory/vipscredentials.xml - for creditem in $(cat /tmp/vicredentials.conf) - do - vcenter=$(echo $creditem | cut -d ";" -f 1) - username=$(echo $creditem | cut -d ";" -f 2) - password=$(echo $creditem | cut -d ";" -f 3) - # /usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml add --server $vcenter --username $username --password $password - /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/CredstoreAdmin.ps1 -credstore /mnt/wfs/inventory/vipscredentials.xml -add -server $vcenter -username $username -password $password - done - rm -f /tmp/vicredentials.conf - # chown www-data:www-data /mnt/wfs/inventory/vipscredentials.xml - fi + # if [ -a "/var/www/.vmware/credstore/vicredentials.xml" ]; then + # openssl des3 -d -salt -in /media/cdrom/conf/vicredentials.conf.ss -out /tmp/vicredentials.conf -pass pass:sexigraf -md md5 + # # /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/CredstoreAdmin.ps1 -createstore -credstore /mnt/wfs/inventory/vipscredentials.xml + # for creditem in $(cat /tmp/vicredentials.conf) + # do + # vcenter=$(echo $creditem | cut -d ";" -f 1) + # username=$(echo $creditem | cut -d ";" -f 2) + # password=$(echo $creditem | cut -d ";" -f 3) + # # /usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml add --server $vcenter --username $username --password $password + # /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/CredstoreAdmin.ps1 -credstore /mnt/wfs/inventory/vipscredentials.xml -add -server $vcenter -username $username -password $password + # done + # rm -f /tmp/vicredentials.conf + # # chown www-data:www-data /mnt/wfs/inventory/vipscredentials.xml + # fi if [ -a "/media/cdrom/conf/vipscredentials.xml" ]; then /bin/cp -fR /media/cdrom/conf/vipscredentials.xml /mnt/wfs/inventory/vipscredentials.xml chown www-data:www-data /mnt/wfs/inventory/vipscredentials.xml fi + if [ -a "/media/cdrom/conf/vbrpscredentials.xml" ]; then + /bin/cp -fR /media/cdrom/conf/vbrpscredentials.xml /mnt/wfs/inventory/vbrpscredentials.xml + chown www-data:www-data /mnt/wfs/inventory/vbrpscredentials.xml + fi + # Import cron entries /bin/cp -fR /media/cdrom/conf/cron.d/* /etc/cron.d/ @@ -66,7 +71,11 @@ if [ -d "/mnt/wfs/whisper" ]; then # /bin/sed -i 's/--sessionfile \/tmp\/vpx_/-sessionfile \/tmp\/vmw_/g' /etc/cron.d/vsan_* # /bin/sed -i 's/\.dat$/.key >\/dev\/null 2\>\&1/g' /etc/cron.d/vsan_* - echo "# Virtual SAN has left the building" | tee /etc/cron.d/vsan_* > /dev/null # TODO deal with /etc/cron.d/vsan_* + if compgen -G "/etc/cron.d/vsan_*" > /dev/null; then + echo "# Virtual SAN has left the building" | tee /etc/cron.d/vsan_* + fi + + # TODO rm -rf /mnt/wfs/whisper/vsan/*/*/*/esx/vsan /bin/sed -i 's/\/usr\/bin\/perl \/root\/ViPullStatistics\.pl --credstore \/var\/www\/\.vmware\/credstore\/vicredentials\.xml --server/\/usr\/bin\/pwsh -NonInteractive -NoProfile -f \/opt\/sexigraf\/ViPullStatistics\.ps1 -credstore \/mnt\/wfs\/inventory\/vipscredentials\.xml -server/g' /etc/cron.d/vi_* /bin/sed -i 's/--sessionfile \/tmp\/vpx_/-sessionfile \/tmp\/vmw_/g' /etc/cron.d/vi_* @@ -80,9 +89,9 @@ if [ -d "/mnt/wfs/whisper" ]; then /etc/init.d/grafana-server restart apachectl restart - # 0.99g storage schema change #217 - # nohup /usr/bin/find -L /mnt/wfs/whisper/vmw/ -type f \( -name '*.wsp' \) -exec /usr/local/bin/whisper-resize.py {} 5m:24h 10m:48h 80m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup \; > /var/log/sexigraf/vmw_wsp_resize.log 2>&1 & - # nohup /usr/bin/find -L /mnt/wfs/whisper/esx/ -type f \( -name '*.wsp' \) -exec /usr/local/bin/whisper-resize.py {} 5m:24h 10m:48h 80m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup \; > /var/log/sexigraf/esx_wsp_resize.log 2>&1 & + # 0.99j storage schema change #351 + # nohup /usr/bin/find -L /mnt/wfs/whisper/vmw/ -type f \( -name '*.wsp' \) -exec /usr/local/bin/whisper-resize.py {} 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup \; > /var/log/sexigraf/vmw_wsp_resize.log 2>&1 & + # nohup /usr/bin/find -L /mnt/wfs/whisper/esx/ -type f \( -name '*.wsp' \) -exec /usr/local/bin/whisper-resize.py {} 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup \; > /var/log/sexigraf/esx_wsp_resize.log 2>&1 & # Inventory Update nohup /bin/bash /var/www/scripts/updateInventory.sh & diff --git a/var/www/scripts/removeVbrCrontab.sh b/var/www/scripts/removeVbrCrontab.sh new file mode 100644 index 00000000..b81273e8 --- /dev/null +++ b/var/www/scripts/removeVbrCrontab.sh @@ -0,0 +1,4 @@ +#!/bin/bash +crontabFile="/etc/cron.d/vbr_$(sed s/\\./_/g <<<$1)" +rm -f $crontabFile +service cron reload