From 3368e2d06c2bc902b9e909727139da1dead323a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Mon, 9 Jan 2023 13:10:27 +0100 Subject: [PATCH 001/201] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 562305c9..00a348b1 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ The metrics used in the various VSAN dashboards are now collected every 5 minute Fast. Very fast. That’s what we had in mind when we designed SexiGraf. When you need vSphere metrics, the obvious way is the `PerformanceManager`, but we need something faster so we choosed managed object properties and quickstats like `ResourcePoolQuickStats`. If we have no other choice, we failback to the `PerformanceManager` but we only query the last 15 samples of the `RealTime samplingPeriod` since we pull vSphere metrics every 5 minutes. http://www.sexigraf.fr/vsphere-sexipanels/   -## FreeNAS +## FreeNAS/TrueNAS -Starting from version 9.10, FreeNAS allows users to set a “Remote Graphite Server” target to send all the metrics harvested by Collectd. Guess what would make a nice Graphite target! http://www.sexigraf.fr/freenas-sexipanel/ +Starting from version 9.10, FreeNAS/TrueNAS allows users to set a “Remote Graphite Server” target to send all the metrics harvested by Collectd. Guess what would make a nice Graphite target! http://www.sexigraf.fr/freenas-sexipanel/ ## Windows From b4812fc71c28a551efa4b88b3e851199883104d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 11 Jan 2023 00:51:07 +0100 Subject: [PATCH 002/201] Update ViOfflineInventory.ps1 #325 --- opt/sexigraf/ViOfflineInventory.ps1 | 73 +++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 6eb33409..b8756fca 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.69" +$ScriptVersion = "0.9.70" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -365,7 +365,8 @@ if ($ViServersList.count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory files ..." $ViVmsInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViVmInventory.csv -Force } catch { - AltAndCatchFire "VM Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] VM Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } } @@ -374,7 +375,8 @@ if ($ViServersList.count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Esx Inventory files ..." $ViEsxsInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force } catch { - AltAndCatchFire "ESX Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } } @@ -383,10 +385,73 @@ if ($ViServersList.count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Datastore Inventory files ..." $ViDatastoresInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViDsInventory.csv -Force } catch { - AltAndCatchFire "Datastore Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] Datastore Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } } + Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning vm folders ..." + + $vmfolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + + Write-Host "$((Get-Date).ToString("o")) [INFO] Building vm folders table ..." + + $vmfolders_h = @{} + foreach ($vmfolder in $vmfolders) { + if (!$vmfolders_h[$vmfolder.basename]) { + $vmfolders_h.add($vmfolder.basename,@($vmfolder)) + } else { + $vmfolders_h[$vmfolder.basename] += $vmfolder + } + } + + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xvmotioned vms aka DstVmMigratedEvent ..." + + try { + $vmfoldersdup = (compare-object $vmfolders.basename $($vmfolders|Select-Object basename -unique).basename|?{$_.SideIndicator -eq "<="}).InputObject + } catch { + AltAndCatchFire "Compare folders issue" + } + + if ($vmfoldersdup) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated vm folders found across clusters, evaluating mobility ..." + foreach ($vmdup in $vmfoldersdup) { + if ($vmfolders_h[$vmdup].count -eq 2) { + + $vmdupfolders = $vmfolders_h[$vmdup] + $vmdupsrcdir = $vmdupfolders|Sort-Object CreationTimeUtc -Descending|Select-Object -Last 1 + $vmdupdstdir = $vmdupfolders|Sort-Object CreationTimeUtc -Descending|Select-Object -First 1 + + try { + $vmdupsrcwsp = Get-Item $($vmdupsrcdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $vmdupdstwsp = Get-Item $($vmdupdstdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for vm $vmdup ..." + continue + } + + if (($vmdupdstdir.CreationTimeUtc -gt $vmdupsrcdir.CreationTimeUtc) -and (($vmdupdstwsp.LastWriteTimeUtc - $vmdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 15) -and (($vmdupdstwsp.LastWriteTimeUtc - $vmdupsrcwsp.LastWriteTimeUtc).TotalMinutes -le 90)) { + $vmdupsrcclu = $vmdupsrcdir.FullName.split("/")[-3] + $vmdupdstclu = $vmdupdstdir.FullName.split("/")[-3] + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup has recently been moved from cluster $vmdupsrcclu to cluster $vmdupdstclu, moving metrics to the new destination ..." + try { + Move-Item $vmdupsrcdir.FullName $vmdupdstdir.FullName -Force + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] moving issue for vm $vmdup ..." + continue + } + } + + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] Too many folders for vm $vmdup ..." + } + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated vm folders found" + } + + Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf has left the building ..." + } else { AltAndCatchFire "No VI server to process" } \ No newline at end of file From 1223c22b7dcae5fa8c14c54e8d876f0940bf59a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 11 Jan 2023 00:56:51 +0100 Subject: [PATCH 003/201] Update pfSense.json --- var/lib/grafana/dashboards/Miscellaneous/pfSense.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/Miscellaneous/pfSense.json b/var/lib/grafana/dashboards/Miscellaneous/pfSense.json index 9a309f41..1828a167 100644 --- a/var/lib/grafana/dashboards/Miscellaneous/pfSense.json +++ b/var/lib/grafana/dashboards/Miscellaneous/pfSense.json @@ -4,7 +4,7 @@ ] }, "description": "", - "editable": true, + "editable": false, "gnetId": null, "graphTooltip": 1, "id": null, From b4acbde77b26c7852afed37f6c85d90c64ed2be1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 11 Jan 2023 00:56:58 +0100 Subject: [PATCH 004/201] Update VMware_vSAN_SMART_Stats.json --- .../grafana/dashboards/VMware_vSAN/VMware_vSAN_SMART_Stats.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_vSAN/VMware_vSAN_SMART_Stats.json b/var/lib/grafana/dashboards/VMware_vSAN/VMware_vSAN_SMART_Stats.json index 77c82565..62127b6c 100644 --- a/var/lib/grafana/dashboards/VMware_vSAN/VMware_vSAN_SMART_Stats.json +++ b/var/lib/grafana/dashboards/VMware_vSAN/VMware_vSAN_SMART_Stats.json @@ -3,7 +3,7 @@ "list": [ ] }, - "editable": true, + "editable": false, "gnetId": null, "graphTooltip": 1, "id": null, From 4c3e5ac5813f856b1d0748265593b53b0df4148b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 12 Jan 2023 21:27:48 +0100 Subject: [PATCH 005/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 30 +++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index b8756fca..1b7a764a 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.70" +$ScriptVersion = "0.9.71" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -394,28 +394,24 @@ if ($ViServersList.count -gt 0) { $vmfolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - Write-Host "$((Get-Date).ToString("o")) [INFO] Building vm folders table ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xvmotioned vms aka DstVmMigratedEvent ..." $vmfolders_h = @{} + $vmfoldersdup_h = @{} foreach ($vmfolder in $vmfolders) { if (!$vmfolders_h[$vmfolder.basename]) { $vmfolders_h.add($vmfolder.basename,@($vmfolder)) } else { $vmfolders_h[$vmfolder.basename] += $vmfolder + if (!$vmfoldersdup_h[$vmfolder.basename]) { + $vmfoldersdup_h.add($vmfolder.basename,"1") + } } } - - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xvmotioned vms aka DstVmMigratedEvent ..." - - try { - $vmfoldersdup = (compare-object $vmfolders.basename $($vmfolders|Select-Object basename -unique).basename|?{$_.SideIndicator -eq "<="}).InputObject - } catch { - AltAndCatchFire "Compare folders issue" - } - if ($vmfoldersdup) { + if ($vmfoldersdup_h) { Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated vm folders found across clusters, evaluating mobility ..." - foreach ($vmdup in $vmfoldersdup) { + foreach ($vmdup in $vmfoldersdup_h.keys) { if ($vmfolders_h[$vmdup].count -eq 2) { $vmdupfolders = $vmfolders_h[$vmdup] @@ -440,6 +436,16 @@ if ($ViServersList.count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [EROR] moving issue for vm $vmdup ..." continue } + } elseif (($vmdupdstdir.CreationTimeUtc -gt $vmdupsrcdir.CreationTimeUtc) -and (($vmdupdstwsp.LastWriteTimeUtc - $vmdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 15) -and (($vmdupdstwsp.LastWriteTimeUtc - $vmdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 90)) { + $vmdupsrcclu = $vmdupsrcdir.FullName.split("/")[-3] + $vmdupdstclu = $vmdupdstdir.FullName.split("/")[-3] + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup has been moved from cluster $vmdupsrcclu to cluster $vmdupdstclu a while ago, merging metrics to the new destination ..." + try { + # Move-Item $vmdupsrcdir.FullName $vmdupdstdir.FullName -Force + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] moving issue for vm $vmdup ..." + continue + } } } else { From bf2389add4f4498c97a6bde1cd84d09bbbe45eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 14 Jan 2023 17:41:27 +0100 Subject: [PATCH 006/201] Update ViOfflineInventory.ps1 #325 --- opt/sexigraf/ViOfflineInventory.ps1 | 149 ++++++++++++++++++++++------ 1 file changed, 117 insertions(+), 32 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 1b7a764a..93a6359f 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -1,9 +1,9 @@ -#!/usr/bin/pwsh -Command[EROR] +#!/usr/bin/pwsh -Command # param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.71" +$ScriptVersion = "0.9.72" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -363,7 +363,7 @@ if ($ViServersList.count -gt 0) { if ($ViVmsInfos) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory files ..." - $ViVmsInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViVmInventory.csv -Force + $ViVmsInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViVmInventory.csv -Force -ErrorAction Stop } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] VM Export-Csv issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -373,7 +373,7 @@ if ($ViServersList.count -gt 0) { if ($ViEsxsInfos) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Esx Inventory files ..." - $ViEsxsInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force + $ViEsxsInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force -ErrorAction Stop } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] ESX Export-Csv issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -383,7 +383,7 @@ if ($ViServersList.count -gt 0) { if ($ViDatastoresInfos) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Datastore Inventory files ..." - $ViDatastoresInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViDsInventory.csv -Force + $ViDatastoresInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViDsInventory.csv -Force -ErrorAction Stop } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] Datastore Export-Csv issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -392,7 +392,7 @@ if ($ViServersList.count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning vm folders ..." - $vmfolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $vmfolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xvmotioned vms aka DstVmMigratedEvent ..." @@ -411,49 +411,134 @@ if ($ViServersList.count -gt 0) { if ($vmfoldersdup_h) { Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated vm folders found across clusters, evaluating mobility ..." - foreach ($vmdup in $vmfoldersdup_h.keys) { - if ($vmfolders_h[$vmdup].count -eq 2) { + foreach ($vmdup in $($vmfoldersdup_h.keys|Select-Object -first 500)) { - $vmdupfolders = $vmfolders_h[$vmdup] - $vmdupsrcdir = $vmdupfolders|Sort-Object CreationTimeUtc -Descending|Select-Object -Last 1 - $vmdupdstdir = $vmdupfolders|Sort-Object CreationTimeUtc -Descending|Select-Object -First 1 + if ($vmfolders_h[$vmdup].count -lt 2) { + Write-Host "$((Get-Date).ToString("o")) [EROR] VM $vmdup has less than 2 copies, skipping ..." + continue + } elseif ($vmfolders_h[$vmdup].count -gt 2) { + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup has more than 2 copies ..." + } + + $vmdupfolders = $vmfolders_h[$vmdup] + $vmdupsrcdir = $($vmdupfolders|Sort-Object CreationTimeUtc -Descending)[1] + $vmdupdstdir = $($vmdupfolders|Sort-Object CreationTimeUtc -Descending)[0] + + try { + $vmdupsrcwsp = Get-Item $($vmdupsrcdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $vmdupdstwsp = Get-Item $($vmdupdstdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for vm $vmdup ..." + continue + } + + if (($vmdupdstdir.CreationTimeUtc -gt $vmdupsrcdir.CreationTimeUtc) -and ($vmdupdstwsp.LastWriteTimeUtc - $vmdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 90) { + $vmdupsrcclu = $vmdupsrcdir.FullName.split("/")[-3] + $vmdupdstclu = $vmdupdstdir.FullName.split("/")[-3] + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup has been moved from cluster $vmdupsrcclu to cluster $vmdupdstclu a while ago, merging metrics to the new destination if possible ..." + $vmdupwsps2mv = Get-ChildItem -Recurse $vmdupsrcdir.FullName -Filter *.wsp + foreach ($vmdupwsp2mv in $vmdupwsps2mv) { + if (Test-Path $($vmdupdstdir.FullName + "/" + $vmdupwsp2mv.FullName.split("/")[-2] + "/" + $vmdupwsp2mv.FullName.split("/")[-1])) { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($vmdupwsp2mv.FullName) to $($vmdupdstdir.FullName + "/" + $($vmdupwsp2mv.FullName.split("/")[-2]) + "/" + $($vmdupwsp2mv.FullName.split("/")[-1]))" + $vmdupwspsrcresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($vmdupdstdir.FullName + "/" + $($vmdupwsp2mv.FullName.split("/")[-2]) + "/" + $($vmdupwsp2mv.FullName.split("/")[-1])) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $vmdupwsp2mvresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($vmdupwsp2mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $vmdupwsp2mvmerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($vmdupwsp2mv.FullName) $($vmdupdstdir.FullName + "/" + $($vmdupwsp2mv.FullName.split("/")[-2]) + "/" + $($vmdupwsp2mv.FullName.split("/")[-1]))" + + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($vmdupwsp2mv.FullName) moving issue ..." + continue + } + } else { + if (Test-Path $($vmdupdstdir.FullName + "/" + $vmdupwsp2mv.FullName.split("/")[-2] + "/")) { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Moving $($vmdupwsp2mv.FullName) to $($vmdupdstdir.FullName + "/" + $($vmdupwsp2mv.FullName.split("/")[-2]) + "/")" + $vmdupwspmv = Move-Item $vmdupwsp2mv.FullName $($vmdupdstdir.FullName + "/" + $vmdupwsp2mv.FullName.split("/")[-2] + "/" + $vmdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($vmdupwsp2mv.FullName) moving issue ..." + continue + } + } else { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $($vmdupdstdir.FullName + "/" + $($vmdupwsp2mv.FullName.split("/")[-2]) + "/") and moving $($vmdupwsp2mv.FullName)" + $vmdupwspmkdir = New-Item $($vmdupdstdir.FullName + "/" + $vmdupwsp2mv.FullName.split("/")[-2] + "/") -Force -ErrorAction Stop + $vmdupwspmv = Move-Item $vmdupwsp2mv.FullName $($vmdupdstdir.FullName + "/" + $vmdupwsp2mv.FullName.split("/")[-2] + "/" + $vmdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($vmdupwsp2mv.FullName) moving issue ..." + continue + } + } + } + } + Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($vmdupsrcdir.FullName)" + try { + Remove-Item -Recurse $($vmdupsrcdir.FullName) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($vmdupsrcdir.FullName) issue ..." + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup move is too recent or has clone ..." + } + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated vm folders found" + } + + Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning esx folders ..." + + $esxfolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for esx move across clusters ..." + + $esxfolders_h = @{} + $esxfoldersdup_h = @{} + foreach ($esxfolder in $esxfolders) { + if (!$esxfolders_h[$esxfolder.basename]) { + $esxfolders_h.add($esxfolder.basename,@($esxfolder)) + } else { + $esxfolders_h[$esxfolder.basename] += $esxfolder + if (!$esxfoldersdup_h[$esxfolder.basename]) { + $esxfoldersdup_h.add($esxfolder.basename,"1") + } + } + } + + if ($esxfoldersdup_h) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated esx folders found across clusters, evaluating mobility ..." + foreach ($esxdup in $esxfoldersdup_h.keys|Select-Object -first 100) { + if ($esxfolders_h[$esxdup].count -eq 2) { + + $esxdupfolders = $esxfolders_h[$esxdup] + $esxdupsrcdir = $esxdupfolders|Sort-Object CreationTimeUtc -Descending|Select-Object -Last 1 + $esxdupdstdir = $esxdupfolders|Sort-Object CreationTimeUtc -Descending|Select-Object -First 1 try { - $vmdupsrcwsp = Get-Item $($vmdupsrcdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - $vmdupdstwsp = Get-Item $($vmdupdstdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $esxdupsrcwsp = Get-Item $($esxdupsrcdir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $esxdupdstwsp = Get-Item $($esxdupdstdir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for vm $vmdup ..." + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for esx $esxdup ..." continue } - if (($vmdupdstdir.CreationTimeUtc -gt $vmdupsrcdir.CreationTimeUtc) -and (($vmdupdstwsp.LastWriteTimeUtc - $vmdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 15) -and (($vmdupdstwsp.LastWriteTimeUtc - $vmdupsrcwsp.LastWriteTimeUtc).TotalMinutes -le 90)) { - $vmdupsrcclu = $vmdupsrcdir.FullName.split("/")[-3] - $vmdupdstclu = $vmdupdstdir.FullName.split("/")[-3] - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup has recently been moved from cluster $vmdupsrcclu to cluster $vmdupdstclu, moving metrics to the new destination ..." + if (($esxdupdstdir.CreationTimeUtc -gt $esxdupsrcdir.CreationTimeUtc) -and (($esxdupdstwsp.LastWriteTimeUtc - $esxdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 60)) { + # if (($esxdupdstdir.CreationTimeUtc -gt $esxdupsrcdir.CreationTimeUtc) -and (($esxdupdstwsp.LastWriteTimeUtc - $esxdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 15) -and (($esxdupdstwsp.LastWriteTimeUtc - $esxdupsrcwsp.LastWriteTimeUtc).TotalMinutes -le 90)) { + $esxdupsrcclu = $esxdupsrcdir.FullName.split("/")[-3] + $esxdupdstclu = $esxdupdstdir.FullName.split("/")[-3] + Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup has recently been moved from cluster $esxdupsrcclu to cluster $esxdupdstclu, moving metrics to the new destination ..." try { - Move-Item $vmdupsrcdir.FullName $vmdupdstdir.FullName -Force + # Move-Item $esxdupsrcdir.FullName $esxdupdstdir.FullName -Force } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] moving issue for vm $vmdup ..." - continue - } - } elseif (($vmdupdstdir.CreationTimeUtc -gt $vmdupsrcdir.CreationTimeUtc) -and (($vmdupdstwsp.LastWriteTimeUtc - $vmdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 15) -and (($vmdupdstwsp.LastWriteTimeUtc - $vmdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 90)) { - $vmdupsrcclu = $vmdupsrcdir.FullName.split("/")[-3] - $vmdupdstclu = $vmdupdstdir.FullName.split("/")[-3] - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup has been moved from cluster $vmdupsrcclu to cluster $vmdupdstclu a while ago, merging metrics to the new destination ..." - try { - # Move-Item $vmdupsrcdir.FullName $vmdupdstdir.FullName -Force - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] moving issue for vm $vmdup ..." + Write-Host "$((Get-Date).ToString("o")) [EROR] moving issue for esx $esxdup ..." continue } } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] Too many folders for vm $vmdup ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] Too many folders for esx $esxdup ..." } } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated vm folders found" + Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated esx folders found" } Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf has left the building ..." From db5be7b6382819e03beda671a05398e7c35702d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 14 Jan 2023 18:01:52 +0100 Subject: [PATCH 007/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 147 +++++++++++++++++----------- 1 file changed, 92 insertions(+), 55 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 93a6359f..4ffb39f6 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -477,69 +477,106 @@ if ($ViServersList.count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($vmdupsrcdir.FullName) issue ..." } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup move is too recent or has clone ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup move is too recent or has clones ..." } } } else { Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated vm folders found" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning esx folders ..." + # Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning esx folders ..." - $esxfolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for esx move across clusters ..." - - $esxfolders_h = @{} - $esxfoldersdup_h = @{} - foreach ($esxfolder in $esxfolders) { - if (!$esxfolders_h[$esxfolder.basename]) { - $esxfolders_h.add($esxfolder.basename,@($esxfolder)) - } else { - $esxfolders_h[$esxfolder.basename] += $esxfolder - if (!$esxfoldersdup_h[$esxfolder.basename]) { - $esxfoldersdup_h.add($esxfolder.basename,"1") - } - } - } + # $esxfolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending - if ($esxfoldersdup_h) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated esx folders found across clusters, evaluating mobility ..." - foreach ($esxdup in $esxfoldersdup_h.keys|Select-Object -first 100) { - if ($esxfolders_h[$esxdup].count -eq 2) { - - $esxdupfolders = $esxfolders_h[$esxdup] - $esxdupsrcdir = $esxdupfolders|Sort-Object CreationTimeUtc -Descending|Select-Object -Last 1 - $esxdupdstdir = $esxdupfolders|Sort-Object CreationTimeUtc -Descending|Select-Object -First 1 - - try { - $esxdupsrcwsp = Get-Item $($esxdupsrcdir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - $esxdupdstwsp = Get-Item $($esxdupdstdir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for esx $esxdup ..." - continue - } - - if (($esxdupdstdir.CreationTimeUtc -gt $esxdupsrcdir.CreationTimeUtc) -and (($esxdupdstwsp.LastWriteTimeUtc - $esxdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 60)) { - # if (($esxdupdstdir.CreationTimeUtc -gt $esxdupsrcdir.CreationTimeUtc) -and (($esxdupdstwsp.LastWriteTimeUtc - $esxdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 15) -and (($esxdupdstwsp.LastWriteTimeUtc - $esxdupsrcwsp.LastWriteTimeUtc).TotalMinutes -le 90)) { - $esxdupsrcclu = $esxdupsrcdir.FullName.split("/")[-3] - $esxdupdstclu = $esxdupdstdir.FullName.split("/")[-3] - Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup has recently been moved from cluster $esxdupsrcclu to cluster $esxdupdstclu, moving metrics to the new destination ..." - try { - # Move-Item $esxdupsrcdir.FullName $esxdupdstdir.FullName -Force - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] moving issue for esx $esxdup ..." - continue - } - } - - } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] Too many folders for esx $esxdup ..." - } - } - } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated esx folders found" - } + # Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for moved esx ..." + + # $esxfolders_h = @{} + # $esxfoldersdup_h = @{} + # foreach ($esxfolder in $esxfolders) { + # if (!$esxfolders_h[$esxfolder.basename]) { + # $esxfolders_h.add($esxfolder.basename,@($esxfolder)) + # } else { + # $esxfolders_h[$esxfolder.basename] += $esxfolder + # if (!$esxfoldersdup_h[$esxfolder.basename]) { + # $esxfoldersdup_h.add($esxfolder.basename,"1") + # } + # } + # } + + # if ($esxfoldersdup_h) { + # Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated esx folders found across clusters, evaluating mobility ..." + # foreach ($esxdup in $($esxfoldersdup_h.keys|Select-Object -first 500)) { + + # if ($esxfolders_h[$esxdup].count -lt 2) { + # Write-Host "$((Get-Date).ToString("o")) [EROR] esx $esxdup has less than 2 copies, skipping ..." + # continue + # } elseif ($esxfolders_h[$esxdup].count -gt 2) { + # Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup has more than 2 copies ..." + # } + + # $esxdupfolders = $esxfolders_h[$esxdup] + # $esxdupsrcdir = $($esxdupfolders|Sort-Object CreationTimeUtc -Descending)[1] + # $esxdupdstdir = $($esxdupfolders|Sort-Object CreationTimeUtc -Descending)[0] + + # try { + # $esxdupsrcwsp = Get-Item $($esxdupsrcdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + # $esxdupdstwsp = Get-Item $($esxdupdstdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + # } catch { + # Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for esx $esxdup ..." + # continue + # } + + # if (($esxdupdstdir.CreationTimeUtc -gt $esxdupsrcdir.CreationTimeUtc) -and ($esxdupdstwsp.LastWriteTimeUtc - $esxdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 90) { + # $esxdupsrcclu = $esxdupsrcdir.FullName.split("/")[-3] + # $esxdupdstclu = $esxdupdstdir.FullName.split("/")[-3] + # Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup has been moved from cluster $esxdupsrcclu to cluster $esxdupdstclu a while ago, merging metrics to the new destination if possible ..." + # $esxdupwsps2mv = Get-ChildItem -Recurse $esxdupsrcdir.FullName -Filter *.wsp + # foreach ($esxdupwsp2mv in $esxdupwsps2mv) { + # if (Test-Path $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1])) { + # try { + # Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($esxdupwsp2mv.FullName) to $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1]))" + # $esxdupwspsrcresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1])) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + # $esxdupwsp2mvresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($esxdupwsp2mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + # $esxdupwsp2mvmerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($esxdupwsp2mv.FullName) $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1]))" + + # } catch { + # Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." + # continue + # } + # } else { + # if (Test-Path $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/")) { + # try { + # Write-Host "$((Get-Date).ToString("o")) [INFO] Moving $($esxdupwsp2mv.FullName) to $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/")" + # $esxdupwspmv = Move-Item $esxdupwsp2mv.FullName $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop + # } catch { + # Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." + # continue + # } + # } else { + # try { + # Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/") and moving $($esxdupwsp2mv.FullName)" + # $esxdupwspmkdir = New-Item $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/") -Force -ErrorAction Stop + # $esxdupwspmv = Move-Item $esxdupwsp2mv.FullName $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop + # } catch { + # Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." + # continue + # } + # } + # } + # } + # Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($esxdupsrcdir.FullName)" + # try { + # Remove-Item -Recurse $($esxdupsrcdir.FullName) -Force -ErrorAction Stop + # } catch { + # Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($esxdupsrcdir.FullName) issue ..." + # } + # } else { + # Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup move is too recent or has clones ..." + # } + # } + # } else { + # Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated esx folders found" + # } Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf has left the building ..." From 225943ac1009feebe96befdaa5c90e9f66d95e6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 14 Jan 2023 19:00:57 +0100 Subject: [PATCH 008/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 168 ++++++++++++++-------------- 1 file changed, 84 insertions(+), 84 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 4ffb39f6..e4f586f1 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -484,99 +484,99 @@ if ($ViServersList.count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated vm folders found" } - # Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning esx folders ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning esx folders ..." - # $esxfolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending + $esxfolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending - # Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for moved esx ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for moved esx ..." - # $esxfolders_h = @{} - # $esxfoldersdup_h = @{} - # foreach ($esxfolder in $esxfolders) { - # if (!$esxfolders_h[$esxfolder.basename]) { - # $esxfolders_h.add($esxfolder.basename,@($esxfolder)) - # } else { - # $esxfolders_h[$esxfolder.basename] += $esxfolder - # if (!$esxfoldersdup_h[$esxfolder.basename]) { - # $esxfoldersdup_h.add($esxfolder.basename,"1") - # } - # } - # } + $esxfolders_h = @{} + $esxfoldersdup_h = @{} + foreach ($esxfolder in $esxfolders) { + if (!$esxfolders_h[$esxfolder.basename]) { + $esxfolders_h.add($esxfolder.basename,@($esxfolder)) + } else { + $esxfolders_h[$esxfolder.basename] += $esxfolder + if (!$esxfoldersdup_h[$esxfolder.basename]) { + $esxfoldersdup_h.add($esxfolder.basename,"1") + } + } + } - # if ($esxfoldersdup_h) { - # Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated esx folders found across clusters, evaluating mobility ..." - # foreach ($esxdup in $($esxfoldersdup_h.keys|Select-Object -first 500)) { + if ($esxfoldersdup_h) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated esx folders found across clusters, evaluating mobility ..." + foreach ($esxdup in $($esxfoldersdup_h.keys|Select-Object -first 500)) { - # if ($esxfolders_h[$esxdup].count -lt 2) { - # Write-Host "$((Get-Date).ToString("o")) [EROR] esx $esxdup has less than 2 copies, skipping ..." - # continue - # } elseif ($esxfolders_h[$esxdup].count -gt 2) { - # Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup has more than 2 copies ..." - # } + if ($esxfolders_h[$esxdup].count -lt 2) { + Write-Host "$((Get-Date).ToString("o")) [EROR] esx $esxdup has less than 2 copies, skipping ..." + continue + } elseif ($esxfolders_h[$esxdup].count -gt 2) { + Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup has more than 2 copies ..." + } - # $esxdupfolders = $esxfolders_h[$esxdup] - # $esxdupsrcdir = $($esxdupfolders|Sort-Object CreationTimeUtc -Descending)[1] - # $esxdupdstdir = $($esxdupfolders|Sort-Object CreationTimeUtc -Descending)[0] + $esxdupfolders = $esxfolders_h[$esxdup] + $esxdupsrcdir = $($esxdupfolders|Sort-Object CreationTimeUtc -Descending)[1] + $esxdupdstdir = $($esxdupfolders|Sort-Object CreationTimeUtc -Descending)[0] - # try { - # $esxdupsrcwsp = Get-Item $($esxdupsrcdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - # $esxdupdstwsp = Get-Item $($esxdupdstdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - # } catch { - # Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for esx $esxdup ..." - # continue - # } + try { + $esxdupsrcwsp = Get-Item $($esxdupsrcdir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $esxdupdstwsp = Get-Item $($esxdupdstdir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for esx $esxdup ..." + continue + } - # if (($esxdupdstdir.CreationTimeUtc -gt $esxdupsrcdir.CreationTimeUtc) -and ($esxdupdstwsp.LastWriteTimeUtc - $esxdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 90) { - # $esxdupsrcclu = $esxdupsrcdir.FullName.split("/")[-3] - # $esxdupdstclu = $esxdupdstdir.FullName.split("/")[-3] - # Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup has been moved from cluster $esxdupsrcclu to cluster $esxdupdstclu a while ago, merging metrics to the new destination if possible ..." - # $esxdupwsps2mv = Get-ChildItem -Recurse $esxdupsrcdir.FullName -Filter *.wsp - # foreach ($esxdupwsp2mv in $esxdupwsps2mv) { - # if (Test-Path $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1])) { - # try { - # Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($esxdupwsp2mv.FullName) to $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1]))" - # $esxdupwspsrcresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1])) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - # $esxdupwsp2mvresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($esxdupwsp2mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - # $esxdupwsp2mvmerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($esxdupwsp2mv.FullName) $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1]))" + if (($esxdupdstdir.CreationTimeUtc -gt $esxdupsrcdir.CreationTimeUtc) -and ($esxdupdstwsp.LastWriteTimeUtc - $esxdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 90) { + $esxdupsrcclu = $esxdupsrcdir.FullName.split("/")[-3] + $esxdupdstclu = $esxdupdstdir.FullName.split("/")[-3] + Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup has been moved from cluster $esxdupsrcclu to cluster $esxdupdstclu a while ago, merging metrics to the new destination if possible ..." + $esxdupwsps2mv = Get-ChildItem -Recurse $esxdupsrcdir.FullName -Filter *.wsp + foreach ($esxdupwsp2mv in $esxdupwsps2mv) { + if (Test-Path $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1])) { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($esxdupwsp2mv.FullName) to $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1]))" + $esxdupwspsrcresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1])) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $esxdupwsp2mvresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($esxdupwsp2mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $esxdupwsp2mvmerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($esxdupwsp2mv.FullName) $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1]))" - # } catch { - # Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." - # continue - # } - # } else { - # if (Test-Path $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/")) { - # try { - # Write-Host "$((Get-Date).ToString("o")) [INFO] Moving $($esxdupwsp2mv.FullName) to $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/")" - # $esxdupwspmv = Move-Item $esxdupwsp2mv.FullName $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop - # } catch { - # Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." - # continue - # } - # } else { - # try { - # Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/") and moving $($esxdupwsp2mv.FullName)" - # $esxdupwspmkdir = New-Item $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/") -Force -ErrorAction Stop - # $esxdupwspmv = Move-Item $esxdupwsp2mv.FullName $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop - # } catch { - # Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." - # continue - # } - # } - # } - # } - # Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($esxdupsrcdir.FullName)" - # try { - # Remove-Item -Recurse $($esxdupsrcdir.FullName) -Force -ErrorAction Stop - # } catch { - # Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($esxdupsrcdir.FullName) issue ..." - # } - # } else { - # Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup move is too recent or has clones ..." - # } - # } - # } else { - # Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated esx folders found" - # } + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." + continue + } + } else { + if (Test-Path $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/")) { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Moving $($esxdupwsp2mv.FullName) to $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/")" + $esxdupwspmv = Move-Item $esxdupwsp2mv.FullName $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." + continue + } + } else { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/") and moving $($esxdupwsp2mv.FullName)" + $esxdupwspmkdir = New-Item $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/") -Force -ErrorAction Stop + $esxdupwspmv = Move-Item $esxdupwsp2mv.FullName $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." + continue + } + } + } + } + Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($esxdupsrcdir.FullName)" + try { + Remove-Item -Recurse $($esxdupsrcdir.FullName) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($esxdupsrcdir.FullName) issue ..." + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup move is too recent or has clones ..." + } + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated esx folders found" + } Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf has left the building ..." From fb693cfcc697559b906f377d0f440cf25b572a1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 14 Jan 2023 23:05:56 +0100 Subject: [PATCH 009/201] Update ViOfflineInventory.ps1 #325 --- opt/sexigraf/ViOfflineInventory.ps1 | 216 +++++++++++++--------------- 1 file changed, 103 insertions(+), 113 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index e4f586f1..fd7c81ac 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -392,190 +392,180 @@ if ($ViServersList.count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning vm folders ..." - $vmfolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending + $VmFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xvmotioned vms aka DstVmMigratedEvent ..." - $vmfolders_h = @{} - $vmfoldersdup_h = @{} - foreach ($vmfolder in $vmfolders) { - if (!$vmfolders_h[$vmfolder.basename]) { - $vmfolders_h.add($vmfolder.basename,@($vmfolder)) + $VmFolders_h = @{} + $VmFoldersDup_h = @{} + foreach ($VmFolder in $VmFolders) { + if (!$VmFolders_h[$VmFolder.basename]) { + $VmFolders_h.add($VmFolder.basename,@($VmFolder)) } else { - $vmfolders_h[$vmfolder.basename] += $vmfolder - if (!$vmfoldersdup_h[$vmfolder.basename]) { - $vmfoldersdup_h.add($vmfolder.basename,"1") + $VmFolders_h[$VmFolder.basename] += $VmFolder + if (!$VmFoldersDup_h[$VmFolder.basename]) { + $VmFoldersDup_h.add($VmFolder.basename,"1") } } } - if ($vmfoldersdup_h) { + if ($VmFoldersDup_h) { Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated vm folders found across clusters, evaluating mobility ..." - foreach ($vmdup in $($vmfoldersdup_h.keys|Select-Object -first 500)) { + foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 500)) { - if ($vmfolders_h[$vmdup].count -lt 2) { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $vmdup has less than 2 copies, skipping ..." + if ($VmFolders_h[$VmDup].count -lt 2) { + Write-Host "$((Get-Date).ToString("o")) [EROR] VM $VmDup has less than 2 copies, skipping ..." continue - } elseif ($vmfolders_h[$vmdup].count -gt 2) { - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup has more than 2 copies ..." + } elseif ($VmFolders_h[$VmDup].count -gt 2) { + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has more than 2 copies ..." } - $vmdupfolders = $vmfolders_h[$vmdup] - $vmdupsrcdir = $($vmdupfolders|Sort-Object CreationTimeUtc -Descending)[1] - $vmdupdstdir = $($vmdupfolders|Sort-Object CreationTimeUtc -Descending)[0] + $VmDupFolders = $VmFolders_h[$VmDup] + $VmDupSrcDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[1] + $VmDupDstDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[0] try { - $vmdupsrcwsp = Get-Item $($vmdupsrcdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - $vmdupdstwsp = Get-Item $($vmdupdstdir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $VmDupSrcWsp = Get-Item $($VmDupSrcDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $VmDupDstWsp = Get-Item $($VmDupDstDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for vm $vmdup ..." + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for vm $VmDup ..." continue } - if (($vmdupdstdir.CreationTimeUtc -gt $vmdupsrcdir.CreationTimeUtc) -and ($vmdupdstwsp.LastWriteTimeUtc - $vmdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 90) { - $vmdupsrcclu = $vmdupsrcdir.FullName.split("/")[-3] - $vmdupdstclu = $vmdupdstdir.FullName.split("/")[-3] - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup has been moved from cluster $vmdupsrcclu to cluster $vmdupdstclu a while ago, merging metrics to the new destination if possible ..." - $vmdupwsps2mv = Get-ChildItem -Recurse $vmdupsrcdir.FullName -Filter *.wsp - foreach ($vmdupwsp2mv in $vmdupwsps2mv) { - if (Test-Path $($vmdupdstdir.FullName + "/" + $vmdupwsp2mv.FullName.split("/")[-2] + "/" + $vmdupwsp2mv.FullName.split("/")[-1])) { + if (($VmDupDstDir.CreationTimeUtc -gt $VmDupSrcDir.CreationTimeUtc) -and ($VmDupDstWsp.LastWriteTimeUtc - $VmDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { + + $VmDupDstVc = $VmDupDstDir.FullName.split("/")[5] + $VmDupDstDc = $VmDupDstDir.FullName.split("/")[6] + $VmDupDstClu = $VmDupDstDir.FullName.split("/")[7] + $VmDupSrcClu = $VmDupSrcDir.FullName.split("/")[7] + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has been moved from cluster $VmDupSrcClu to cluster $VmDupDstClu a while ago, merging metrics to the new destination if possible ..." + $VmDupWsps2Mv = Get-ChildItem -Recurse $VmDupSrcDir.FullName -Filter *.wsp + foreach ($VmDupWsp2Mv in $VmDupWsps2Mv) { + $WspRelativePath = $($VmDupWsp2Mv.FullName -split "/vm/")[1] + $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $VmDupDstVc + "/" + $VmDupDstDc + "/" + $VmDupDstClu + "/vm/" + $WspRelativePath) + if (Test-Path $DstWspFullPath) { try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($vmdupwsp2mv.FullName) to $($vmdupdstdir.FullName + "/" + $($vmdupwsp2mv.FullName.split("/")[-2]) + "/" + $($vmdupwsp2mv.FullName.split("/")[-1]))" - $vmdupwspsrcresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($vmdupdstdir.FullName + "/" + $($vmdupwsp2mv.FullName.split("/")[-2]) + "/" + $($vmdupwsp2mv.FullName.split("/")[-1])) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $vmdupwsp2mvresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($vmdupwsp2mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $vmdupwsp2mvmerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($vmdupwsp2mv.FullName) $($vmdupdstdir.FullName + "/" + $($vmdupwsp2mv.FullName.split("/")[-2]) + "/" + $($vmdupwsp2mv.FullName.split("/")[-1]))" + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($VmDupWsp2Mv.FullName) to $DstWspFullPath" + $VmDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $VmDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($VmDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $VmDupWsp2MvMerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($VmDupWsp2Mv.FullName) $DstWspFullPath" } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($vmdupwsp2mv.FullName) moving issue ..." + Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." continue } } else { - if (Test-Path $($vmdupdstdir.FullName + "/" + $vmdupwsp2mv.FullName.split("/")[-2] + "/")) { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Moving $($vmdupwsp2mv.FullName) to $($vmdupdstdir.FullName + "/" + $($vmdupwsp2mv.FullName.split("/")[-2]) + "/")" - $vmdupwspmv = Move-Item $vmdupwsp2mv.FullName $($vmdupdstdir.FullName + "/" + $vmdupwsp2mv.FullName.split("/")[-2] + "/" + $vmdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($vmdupwsp2mv.FullName) moving issue ..." - continue - } - } else { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $($vmdupdstdir.FullName + "/" + $($vmdupwsp2mv.FullName.split("/")[-2]) + "/") and moving $($vmdupwsp2mv.FullName)" - $vmdupwspmkdir = New-Item $($vmdupdstdir.FullName + "/" + $vmdupwsp2mv.FullName.split("/")[-2] + "/") -Force -ErrorAction Stop - $vmdupwspmv = Move-Item $vmdupwsp2mv.FullName $($vmdupdstdir.FullName + "/" + $vmdupwsp2mv.FullName.split("/")[-2] + "/" + $vmdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($vmdupwsp2mv.FullName) moving issue ..." - continue - } + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($VmDupWsp2Mv.FullName)" + $VmDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop + $VmDupWspMv = Move-Item $VmDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." + continue } } } - Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($vmdupsrcdir.FullName)" + Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($VmDupSrcDir.FullName)" try { - Remove-Item -Recurse $($vmdupsrcdir.FullName) -Force -ErrorAction Stop + Remove-Item -Recurse $($VmDupSrcDir.FullName) -Force -ErrorAction Stop } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($vmdupsrcdir.FullName) issue ..." + Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($VmDupSrcDir.FullName) issue ..." } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $vmdup move is too recent or has clones ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup move is too recent, has clones or has come back to its original location ..." } } } else { Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated vm folders found" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning esx folders ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning Esx folders ..." - $esxfolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending + $EsxFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for moved esx ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xEsxotioned Esxs aka DstEsxMigratedEvent ..." - $esxfolders_h = @{} - $esxfoldersdup_h = @{} - foreach ($esxfolder in $esxfolders) { - if (!$esxfolders_h[$esxfolder.basename]) { - $esxfolders_h.add($esxfolder.basename,@($esxfolder)) + $EsxFolders_h = @{} + $EsxFoldersDup_h = @{} + foreach ($EsxFolder in $EsxFolders) { + if (!$EsxFolders_h[$EsxFolder.basename]) { + $EsxFolders_h.add($EsxFolder.basename,@($EsxFolder)) } else { - $esxfolders_h[$esxfolder.basename] += $esxfolder - if (!$esxfoldersdup_h[$esxfolder.basename]) { - $esxfoldersdup_h.add($esxfolder.basename,"1") + $EsxFolders_h[$EsxFolder.basename] += $EsxFolder + if (!$EsxFoldersDup_h[$EsxFolder.basename]) { + $EsxFoldersDup_h.add($EsxFolder.basename,"1") } } } - if ($esxfoldersdup_h) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated esx folders found across clusters, evaluating mobility ..." - foreach ($esxdup in $($esxfoldersdup_h.keys|Select-Object -first 500)) { + if ($EsxFoldersDup_h) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated Esx folders found across clusters, evaluating mobility ..." + foreach ($EsxDup in $($EsxFoldersDup_h.keys|Select-Object -first 500)) { - if ($esxfolders_h[$esxdup].count -lt 2) { - Write-Host "$((Get-Date).ToString("o")) [EROR] esx $esxdup has less than 2 copies, skipping ..." + if ($EsxFolders_h[$EsxDup].count -lt 2) { + Write-Host "$((Get-Date).ToString("o")) [EROR] Esx $EsxDup has less than 2 copies, skipping ..." continue - } elseif ($esxfolders_h[$esxdup].count -gt 2) { - Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup has more than 2 copies ..." + } elseif ($EsxFolders_h[$EsxDup].count -gt 2) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has more than 2 copies ..." } - $esxdupfolders = $esxfolders_h[$esxdup] - $esxdupsrcdir = $($esxdupfolders|Sort-Object CreationTimeUtc -Descending)[1] - $esxdupdstdir = $($esxdupfolders|Sort-Object CreationTimeUtc -Descending)[0] + $EsxDupFolders = $EsxFolders_h[$EsxDup] + $EsxDupSrcDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[1] + $EsxDupDstDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[0] try { - $esxdupsrcwsp = Get-Item $($esxdupsrcdir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - $esxdupdstwsp = Get-Item $($esxdupdstdir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $EsxDupSrcWsp = Get-Item $($EsxDupSrcDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $EsxDupDstWsp = Get-Item $($EsxDupDstDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for esx $esxdup ..." + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for Esx $EsxDup ..." continue } - if (($esxdupdstdir.CreationTimeUtc -gt $esxdupsrcdir.CreationTimeUtc) -and ($esxdupdstwsp.LastWriteTimeUtc - $esxdupsrcwsp.LastWriteTimeUtc).TotalMinutes -gt 90) { - $esxdupsrcclu = $esxdupsrcdir.FullName.split("/")[-3] - $esxdupdstclu = $esxdupdstdir.FullName.split("/")[-3] - Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup has been moved from cluster $esxdupsrcclu to cluster $esxdupdstclu a while ago, merging metrics to the new destination if possible ..." - $esxdupwsps2mv = Get-ChildItem -Recurse $esxdupsrcdir.FullName -Filter *.wsp - foreach ($esxdupwsp2mv in $esxdupwsps2mv) { - if (Test-Path $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1])) { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($esxdupwsp2mv.FullName) to $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1]))" - $esxdupwspsrcresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1])) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $esxdupwsp2mvresiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($esxdupwsp2mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $esxdupwsp2mvmerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($esxdupwsp2mv.FullName) $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/" + $($esxdupwsp2mv.FullName.split("/")[-1]))" + if (($EsxDupDstDir.CreationTimeUtc -gt $EsxDupSrcDir.CreationTimeUtc) -and ($EsxDupDstWsp.LastWriteTimeUtc - $EsxDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { + $EsxDupDstVc = $EsxDupDstDir.FullName.split("/")[5] + $EsxDupDstDc = $EsxDupDstDir.FullName.split("/")[6] + $EsxDupDstClu = $EsxDupDstDir.FullName.split("/")[7] + $EsxDupSrcClu = $EsxDupSrcDir.FullName.split("/")[7] + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has been moved from cluster $EsxDupSrcClu to cluster $EsxDupDstClu a while ago, merging metrics to the new destination if possible ..." + $EsxDupWsps2Mv = Get-ChildItem -Recurse $EsxDupSrcDir.FullName -Filter *.wsp + foreach ($EsxDupWsp2Mv in $EsxDupWsps2Mv) { + $WspRelativePath = $($EsxDupWsp2Mv.FullName -split "/Esx/")[1] + $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $EsxDupDstVc + "/" + $EsxDupDstDc + "/" + $EsxDupDstClu + "/esx/" + $WspRelativePath) + if (Test-Path $DstWspFullPath) { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($EsxDupWsp2Mv.FullName) to $DstWspFullPath" + $EsxDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $EsxDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($EsxDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $EsxDupWsp2MEsxerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($EsxDupWsp2Mv.FullName) $DstWspFullPath" + } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." + Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." continue } } else { - if (Test-Path $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/")) { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Moving $($esxdupwsp2mv.FullName) to $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/")" - $esxdupwspmv = Move-Item $esxdupwsp2mv.FullName $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." - continue - } - } else { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $($esxdupdstdir.FullName + "/" + $($esxdupwsp2mv.FullName.split("/")[-2]) + "/") and moving $($esxdupwsp2mv.FullName)" - $esxdupwspmkdir = New-Item $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/") -Force -ErrorAction Stop - $esxdupwspmv = Move-Item $esxdupwsp2mv.FullName $($esxdupdstdir.FullName + "/" + $esxdupwsp2mv.FullName.split("/")[-2] + "/" + $esxdupwsp2mv.FullName.split("/")[-1]) -Force -ErrorAction Stop - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($esxdupwsp2mv.FullName) moving issue ..." - continue - } + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($EsxDupWsp2Mv.FullName)" + $EsxDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop + $EsxDupWspMv = Move-Item $EsxDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." + continue } } } - Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($esxdupsrcdir.FullName)" + Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($EsxDupSrcDir.FullName)" try { - Remove-Item -Recurse $($esxdupsrcdir.FullName) -Force -ErrorAction Stop + Remove-Item -Recurse $($EsxDupSrcDir.FullName) -Force -ErrorAction Stop } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($esxdupsrcdir.FullName) issue ..." + Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($EsxDupSrcDir.FullName) issue ..." } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] esx $esxdup move is too recent or has clones ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup move is too recent, has clones or has come back to its original location ..." } } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated esx folders found" + Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated Esx folders found" } Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf has left the building ..." From 0dad9e8f6f3397fab925aa24025cdb74090f2247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 14 Jan 2023 23:56:34 +0100 Subject: [PATCH 010/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index fd7c81ac..7e2cdcea 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -500,7 +500,7 @@ if ($ViServersList.count -gt 0) { if ($EsxFoldersDup_h) { Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated Esx folders found across clusters, evaluating mobility ..." - foreach ($EsxDup in $($EsxFoldersDup_h.keys|Select-Object -first 500)) { + foreach ($EsxDup in $($EsxFoldersDup_h.keys|Select-Object -first 50)) { if ($EsxFolders_h[$EsxDup].count -lt 2) { Write-Host "$((Get-Date).ToString("o")) [EROR] Esx $EsxDup has less than 2 copies, skipping ..." From 0fac7e4e03bead538f03a6688488afc358130980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 18 Jan 2023 22:36:13 +0100 Subject: [PATCH 011/201] #284 + net iops --- opt/sexigraf/ViPullStatistics.ps1 | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 46645c20..846fe7b7 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1006" +$ScriptVersion = "0.9.1007" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -500,8 +500,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # $VmMultiMetricsAll = @( # "virtualdisk.numberWriteAveraged.average", # "virtualdisk.numberReadAveraged.average", - # "net.received.average", - # "net.transmitted.average" + # "net.packetsRx.summation", + # "net.packetsTx.summation" # ) # try { @@ -533,8 +533,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # $VmMultiMetricsAll = @( # "virtualdisk.numberWriteAveraged.average", # "virtualdisk.numberReadAveraged.average", - # "net.received.average", - # "net.transmitted.average" + # "net.packetsRx.summation", + # "net.packetsTx.summation" # ) # try { @@ -999,12 +999,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_cluster_vm.moref.value]) { # $vcenter_cluster_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.diskiops", $vcenter_cluster_vm_disk_iops) + # $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.diskIOPS", $vcenter_cluster_vm_disk_iops) # } - # if ($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$vcenter_cluster_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.received.average"]][$vcenter_cluster_vm.moref.value]) { - # $vcenter_cluster_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.received.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.netUsage", $vcenter_cluster_vm_disk_iops) + # if ($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_cluster_vm.moref.value]) { + # $vcenter_cluster_vm_net_iops = $($($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum) / 300 + # $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.netIOPS", $vcenter_cluster_vm_net_iops) # } if ($VmMultiStats[$PerfCounterTable["net.usage.average"]][$vcenter_cluster_vm.moref.value][""]) { @@ -1813,7 +1813,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_standalone_host_vm.moref.value]) { # $vcenter_standalone_host_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.diskiops", $vcenter_standalone_host_vm_disk_iops) + # $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.diskIOPS", $vcenter_standalone_host_vm_disk_iops) + # } + + # if ($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_standalone_host_vm.moref.value]) { + # $vcenter_standalone_host_vm_net_iops = $($($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum) / 300 + # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.netIOPS", $vcenter_standalone_host_vm_net_iops) # } if ($VmMultiStats[$PerfCounterTable["net.usage.average"]][$vcenter_standalone_host_vm.moref.value][""]) { @@ -2024,8 +2029,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # $VmMultiMetricsAll = @( # "virtualdisk.numberWriteAveraged.average", # "virtualdisk.numberReadAveraged.average", - # "net.received.average", - # "net.transmitted.average" + # "net.packetsRx.summation", + # "net.packetsTx.summation" # ) if ($vcenter_vms) { @@ -2044,11 +2049,6 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # } } - - - - - try { $unmanaged_host = $vcenter_vmhosts $unmanaged_compute_resource = $vcenter_clusters @@ -2368,12 +2368,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$unmanaged_host_vm.moref.value]) { # $unmanaged_host_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.diskiops", $unmanaged_host_vm_disk_iops) + # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.diskIOPS", $unmanaged_host_vm_disk_iops) # } - # if ($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$unmanaged_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.received.average"]][$unmanaged_host_vm.moref.value]) { - # $unmanaged_host_vm_net_usage = $($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.received.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.transmitted.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.netUsage", $unmanaged_host_vm_net_usage) + # if ($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$unmanaged_host_vm.moref.value]) { + # $unmanaged_host_vm_net_iops = $($($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum) / 300 + # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.netIOPS", $unmanaged_host_vm_net_iops) # } if ($VmMultiStats[$PerfCounterTable["net.usage.average"]][$unmanaged_host_vm.moref.value][""]) { From 280a165508d30a9f1505b4103f3e9b674d614832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 24 Jan 2023 21:17:27 +0100 Subject: [PATCH 012/201] Update VMware_Cluster_Multi_ESX_LiteStats.json --- .../VMware_Cluster/VMware_Cluster_Multi_ESX_LiteStats.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Multi_ESX_LiteStats.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Multi_ESX_LiteStats.json index f2647232..7c2c16f0 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Multi_ESX_LiteStats.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_Multi_ESX_LiteStats.json @@ -497,7 +497,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, From 27275f514b0f4bab7b5d402992a92c7ebb952029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 24 Jan 2023 21:18:43 +0100 Subject: [PATCH 013/201] Update VMware_VSAN_Resync.json --- var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Resync.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Resync.json b/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Resync.json index 40a01d4d..51b0e5b4 100644 --- a/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Resync.json +++ b/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Resync.json @@ -394,7 +394,7 @@ } ], "maxPerRow": 1, - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, From 14cc4543b6f9a17b161b5dd84434724de73db4df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 24 Jan 2023 21:18:45 +0100 Subject: [PATCH 014/201] Update VMware_All_Datastore_Usage_Distribution.json --- .../VMware_All_Datastore_Usage_Distribution.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_Datastore/VMware_All_Datastore_Usage_Distribution.json b/var/lib/grafana/dashboards/VMware_Datastore/VMware_All_Datastore_Usage_Distribution.json index 8a70e5ed..d1ddf616 100644 --- a/var/lib/grafana/dashboards/VMware_Datastore/VMware_All_Datastore_Usage_Distribution.json +++ b/var/lib/grafana/dashboards/VMware_Datastore/VMware_All_Datastore_Usage_Distribution.json @@ -52,7 +52,7 @@ "lines": true, "linewidth": 0, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, From 090e1b693dd9df1d59138b069d72fb9deabb0e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 24 Jan 2023 21:18:48 +0100 Subject: [PATCH 015/201] Update Graphite_Carbon_Metrics.json --- .../SexiGraf/Graphite_Carbon_Metrics.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json b/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json index e237a664..b9edf10b 100644 --- a/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json +++ b/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json @@ -174,7 +174,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -481,7 +481,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -666,7 +666,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -758,7 +758,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -861,7 +861,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -955,7 +955,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -1049,7 +1049,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -1147,7 +1147,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -1239,7 +1239,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, @@ -1331,7 +1331,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "dataLinks": [] }, From 34b033160add2934d1786d4dc345aa2604862c77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 24 Jan 2023 21:18:50 +0100 Subject: [PATCH 016/201] Update Windows.json --- .../dashboards/Miscellaneous/Windows.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/var/lib/grafana/dashboards/Miscellaneous/Windows.json b/var/lib/grafana/dashboards/Miscellaneous/Windows.json index 5092a73e..cf363d48 100644 --- a/var/lib/grafana/dashboards/Miscellaneous/Windows.json +++ b/var/lib/grafana/dashboards/Miscellaneous/Windows.json @@ -53,7 +53,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -132,7 +132,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -212,7 +212,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -285,7 +285,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -347,7 +347,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -409,7 +409,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -471,7 +471,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -547,7 +547,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -609,7 +609,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, From 544bc90fa8883634d60423487187b053d6627a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 24 Jan 2023 21:18:52 +0100 Subject: [PATCH 017/201] Update pfSense.json --- .../dashboards/Miscellaneous/pfSense.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/var/lib/grafana/dashboards/Miscellaneous/pfSense.json b/var/lib/grafana/dashboards/Miscellaneous/pfSense.json index 1828a167..00223887 100644 --- a/var/lib/grafana/dashboards/Miscellaneous/pfSense.json +++ b/var/lib/grafana/dashboards/Miscellaneous/pfSense.json @@ -63,7 +63,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -159,7 +159,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -271,7 +271,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -394,7 +394,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -510,7 +510,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -614,7 +614,7 @@ "lines": false, "linewidth": 0, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -749,7 +749,7 @@ "linewidth": 2, "links": [], "maxPerRow": 3, - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, @@ -871,7 +871,7 @@ "linewidth": 2, "links": [], "maxPerRow": 3, - "nullPointMode": "connected", + "nullPointMode": "null", "options": { "alertThreshold": true }, From 44e718a3d9016dffbcb2d325e787c5ccc1232c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 24 Jan 2023 21:18:54 +0100 Subject: [PATCH 018/201] Update FreeNAS.json --- .../dashboards/Miscellaneous/FreeNAS.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/var/lib/grafana/dashboards/Miscellaneous/FreeNAS.json b/var/lib/grafana/dashboards/Miscellaneous/FreeNAS.json index cf7c0c3e..a453ecd8 100644 --- a/var/lib/grafana/dashboards/Miscellaneous/FreeNAS.json +++ b/var/lib/grafana/dashboards/Miscellaneous/FreeNAS.json @@ -51,7 +51,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -140,7 +140,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -213,7 +213,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -286,7 +286,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -360,7 +360,7 @@ "lines": true, "linewidth": 1, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -453,7 +453,7 @@ "lines": true, "linewidth": 2, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -535,7 +535,7 @@ "lines": true, "linewidth": 0, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -598,7 +598,7 @@ "lines": true, "linewidth": 0, "links": [], - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -671,7 +671,7 @@ "linewidth": 2, "links": [], "minSpan": 4, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, @@ -761,7 +761,7 @@ "linewidth": 2, "links": [], "minSpan": 4, - "nullPointMode": "connected", + "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, From 730348ab19e8e9e341c6643bb4e1c1b5d51e4348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 28 Jan 2023 23:29:05 +0100 Subject: [PATCH 019/201] Update ViPullStatistics.ps1 #258 --- opt/sexigraf/ViPullStatistics.ps1 | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 846fe7b7..c8327431 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1007" +$ScriptVersion = "0.9.1008" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -313,7 +313,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_datacenters = Get-View -ViewType Datacenter -Property Name, Parent -Server $Server $vcenter_resource_pools = Get-View -ViewType ResourcePool -Property Vm, Parent, Owner, summary.quickStats -Server $Server $vcenter_clusters = Get-View -ViewType ComputeResource -Property name, parent, summary, resourcePool, host, datastore, ConfigurationEx -Server $Server - $vcenter_vmhosts = Get-View -ViewType HostSystem -Property config.network.pnic, config.network.vnic, config.network.dnsConfig.hostName, runtime.connectionState, summary.hardware.numCpuCores, summary.quickStats.distributedCpuFairness, summary.quickStats.distributedMemoryFairness, summary.quickStats.overallCpuUsage, summary.quickStats.overallMemoryUsage, summary.quickStats.uptime, overallStatus, config.storageDevice.hostBusAdapter, vm, name, summary.runtime.healthSystemRuntime.systemHealthInfo.numericSensorInfo, config.product.version, config.product.build, summary.hardware.vendor, summary.hardware.model, summary.hardware.cpuModel, summary.hardware.NumCpuPkgs, Config.VsanHostConfig.ClusterInfo -filter @{"Runtime.ConnectionState" = "^connected$"} -Server $Server + $vcenter_vmhosts = Get-View -ViewType HostSystem -Property config.network.pnic, config.network.vnic, config.network.dnsConfig.hostName, runtime.connectionState, summary.hardware.numCpuCores, summary.quickStats.distributedCpuFairness, summary.quickStats.distributedMemoryFairness, summary.quickStats.overallCpuUsage, summary.quickStats.overallMemoryUsage, summary.quickStats.uptime, overallStatus, config.storageDevice.hostBusAdapter, vm, name, summary.runtime.healthSystemRuntime.systemHealthInfo.numericSensorInfo, config.product.version, config.product.build, summary.hardware.vendor, summary.hardware.model, summary.hardware.cpuModel, summary.hardware.NumCpuPkgs, Config.VsanHostConfig.ClusterInfo, Config.MultipathState -filter @{"Runtime.ConnectionState" = "^connected$"} -Server $Server $vcenter_datastores = Get-View -ViewType Datastore -Property summary, iormConfiguration.enabled, iormConfiguration.statsCollectionEnabled, host -filter @{"summary.accessible" = "true"} -Server $Server $vcenter_pods = Get-View -ViewType StoragePod -Property name, summary, parent, childEntity -Server $Server $vcenter_vms = Get-View -ViewType VirtualMachine -Property name, runtime.maxCpuUsage, runtime.maxMemoryUsage, summary.quickStats.overallCpuUsage, summary.quickStats.overallCpuDemand, summary.quickStats.hostMemoryUsage, summary.quickStats.guestMemoryUsage, summary.quickStats.balloonedMemory, summary.quickStats.compressedMemory, summary.quickStats.swappedMemory, summary.storage.committed, summary.storage.uncommitted, config.hardware.numCPU, layoutEx.file, snapshot, runtime.host, summary.runtime.connectionState, summary.runtime.powerState, summary.config.numVirtualDisks, config.version, config.guestId, config.tools.toolsVersion, summary.quickStats.privateMemory, summary.quickStats.consumedOverheadMemory, summary.quickStats.sharedMemory -filter @{"Summary.Runtime.ConnectionState" = "^connected$"} -Server $Server @@ -652,7 +652,6 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_hosts_power_usage = 0 $vcenter_cluster_hosts_vms_dead = 0 - foreach ($vcenter_cluster_host in $vcenter_vmhosts_h[$vcenter_cluster.Host.value]|?{$_}) { $vcenter_cluster_host_name = $vcenter_cluster_host.config.network.dnsConfig.hostName.ToLower() ### why not $vcenter_cluster_host.name.split(".")[0].ToLower() ? @@ -746,7 +745,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } try { - foreach ($vcenter_cluster_host_vmhba in $vcenter_cluster_host.config.storageDevice.hostBusAdapter) { ### XXX dead paths from config.storageDevice.HostBusAdapter to add + foreach ($vcenter_cluster_host_vmhba in $vcenter_cluster_host.config.storageDevice.hostBusAdapter) { ### TODO filter vmhba based on Config.MultipathState.Path $vcenter_cluster_host_vmhba_name = $vcenter_cluster_host_vmhba.device $vcenter_cluster_host_vmhba_bytesRead = $HostMultiStats[$PerfCounterTable["storageAdapter.read.average"]][$vcenter_cluster_host.moref.value][$vcenter_cluster_host_vmhba_name] $vcenter_cluster_host_vmhba_bytesWrite = $HostMultiStats[$PerfCounterTable["storageAdapter.write.average"]][$vcenter_cluster_host.moref.value][$vcenter_cluster_host_vmhba_name] @@ -763,6 +762,15 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } + try { + foreach ($vcenter_cluster_host_path in $vcenter_cluster_host.Config.MultipathState.Path) { + $vcenter_cluster_h["vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.PathState.$($vcenter_cluster_host_path.PathState)"] ++ + } + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name MultipathState issue in cluster $vcenter_cluster_name" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + try { $vcenter_cluster_host_power = $HostMultiStats[$PerfCounterTable["power.power.average"]][$vcenter_cluster_host.moref.value][""] if ($vcenter_cluster_host_power -ge 0) { @@ -1619,7 +1627,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } try { - foreach ($vcenter_standalone_host_vmhba in $vcenter_standalone_host.config.storageDevice.hostBusAdapter) { ### XXX dead paths from config.storageDevice.HostBusAdapter to add + foreach ($vcenter_standalone_host_vmhba in $vcenter_standalone_host.config.storageDevice.hostBusAdapter) { ### TODO filter vmhba based on Config.MultipathState.Path $vcenter_standalone_host_vmhba_name = $vcenter_standalone_host_vmhba.device $vcenter_standalone_host_vmhba_bytesRead = $HostMultiStats[$PerfCounterTable["storageAdapter.read.average"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_vmhba_name] $vcenter_standalone_host_vmhba_bytesWrite = $HostMultiStats[$PerfCounterTable["storageAdapter.write.average"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_vmhba_name] @@ -1635,6 +1643,15 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } + try { + foreach ($vcenter_standalone_host_path in $vcenter_standalone_host.Config.MultipathState.Path) { + $vcenter_standalone_host_h["esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.superstats.PathState.$($vcenter_standalone_host_path.PathState)"] ++ + } + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_standalone_host_name MultipathState issue in cluster $vcenter_cluster_name" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name vms in datacenter $vcenter_standalone_host_dc_name" $StandaloneResourcePoolPrivateMemory = 0 From bceffd9d393c8cf1ad2a9f4930d16e7011ff3f10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 28 Jan 2023 23:29:25 +0100 Subject: [PATCH 020/201] Update ViOfflineInventory.ps1 #325 --- opt/sexigraf/ViOfflineInventory.ps1 | 169 +++++++++++++++------------- 1 file changed, 93 insertions(+), 76 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 7e2cdcea..67972d47 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.72" +$ScriptVersion = "0.9.73" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -362,8 +362,23 @@ if ($ViServersList.count -gt 0) { if ($ViVmsInfos) { try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory files ..." - $ViVmsInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViVmInventory.csv -Force -ErrorAction Stop + Write-Host "$((Get-Date).ToString("o")) [INFO] Building Vm Inventory CSV ..." + $ViVmsInfosCsv = $ViVmsInfos|Export-Csv -NoTypeInformation -ErrorAction Stop + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for vCenter,Cluster,VM differences with previous inventory file ..." + if (Test-Path /mnt/wfs/inventory/ViVmInventory.csv) { + if ($ViVmsInfosCsvDiff = Compare-Object $(Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop) $ViVmsInfosCsv -Property vCenter,Cluster,VM) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Differences detected with previous inventory file ..." + $VmMigratedScan = $true + } else { + $VmMigratedScan = $false + Write-Host "$((Get-Date).ToString("o")) [INFO] No differences detected with previous inventory file ..." + } + } else { + Write-Host "$((Get-Date).ToString("o")) [EROR] No existing Vm Inventory file" + $VmMigratedScan = $true + } + Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory file ..." + $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.csv -Force -ErrorAction Stop } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] VM Export-Csv issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -390,93 +405,95 @@ if ($ViServersList.count -gt 0) { } } - Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning vm folders ..." + if ($VmMigratedScan -eq $true) { + Write-Host "$((Get-Date).ToString("o")) [INFO] VM Inventory differences detected, ccanning vm folders ..." - $VmFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending + $VmFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xvmotioned vms aka DstVmMigratedEvent ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xvmotioned vms aka DstVmMigratedEvent ..." - $VmFolders_h = @{} - $VmFoldersDup_h = @{} - foreach ($VmFolder in $VmFolders) { - if (!$VmFolders_h[$VmFolder.basename]) { - $VmFolders_h.add($VmFolder.basename,@($VmFolder)) - } else { - $VmFolders_h[$VmFolder.basename] += $VmFolder - if (!$VmFoldersDup_h[$VmFolder.basename]) { - $VmFoldersDup_h.add($VmFolder.basename,"1") + $VmFolders_h = @{} + $VmFoldersDup_h = @{} + foreach ($VmFolder in $VmFolders) { + if (!$VmFolders_h[$VmFolder.basename]) { + $VmFolders_h.add($VmFolder.basename,@($VmFolder)) + } else { + $VmFolders_h[$VmFolder.basename] += $VmFolder + if (!$VmFoldersDup_h[$VmFolder.basename]) { + $VmFoldersDup_h.add($VmFolder.basename,"1") + } } } - } - - if ($VmFoldersDup_h) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated vm folders found across clusters, evaluating mobility ..." - foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 500)) { + + if ($VmFoldersDup_h) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated vm folders found across clusters, evaluating mobility ..." + foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 500)) { - if ($VmFolders_h[$VmDup].count -lt 2) { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $VmDup has less than 2 copies, skipping ..." - continue - } elseif ($VmFolders_h[$VmDup].count -gt 2) { - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has more than 2 copies ..." - } + if ($VmFolders_h[$VmDup].count -lt 2) { + Write-Host "$((Get-Date).ToString("o")) [EROR] VM $VmDup has less than 2 copies, skipping ..." + continue + } elseif ($VmFolders_h[$VmDup].count -gt 2) { + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has more than 2 copies ..." + } - $VmDupFolders = $VmFolders_h[$VmDup] - $VmDupSrcDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[1] - $VmDupDstDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[0] + $VmDupFolders = $VmFolders_h[$VmDup] + $VmDupSrcDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[1] + $VmDupDstDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[0] - try { - $VmDupSrcWsp = Get-Item $($VmDupSrcDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - $VmDupDstWsp = Get-Item $($VmDupDstDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for vm $VmDup ..." - continue - } + try { + $VmDupSrcWsp = Get-Item $($VmDupSrcDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $VmDupDstWsp = Get-Item $($VmDupDstDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for vm $VmDup ..." + continue + } - if (($VmDupDstDir.CreationTimeUtc -gt $VmDupSrcDir.CreationTimeUtc) -and ($VmDupDstWsp.LastWriteTimeUtc - $VmDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { - - $VmDupDstVc = $VmDupDstDir.FullName.split("/")[5] - $VmDupDstDc = $VmDupDstDir.FullName.split("/")[6] - $VmDupDstClu = $VmDupDstDir.FullName.split("/")[7] - $VmDupSrcClu = $VmDupSrcDir.FullName.split("/")[7] - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has been moved from cluster $VmDupSrcClu to cluster $VmDupDstClu a while ago, merging metrics to the new destination if possible ..." - $VmDupWsps2Mv = Get-ChildItem -Recurse $VmDupSrcDir.FullName -Filter *.wsp - foreach ($VmDupWsp2Mv in $VmDupWsps2Mv) { - $WspRelativePath = $($VmDupWsp2Mv.FullName -split "/vm/")[1] - $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $VmDupDstVc + "/" + $VmDupDstDc + "/" + $VmDupDstClu + "/vm/" + $WspRelativePath) - if (Test-Path $DstWspFullPath) { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($VmDupWsp2Mv.FullName) to $DstWspFullPath" - $VmDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $VmDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($VmDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $VmDupWsp2MvMerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($VmDupWsp2Mv.FullName) $DstWspFullPath" - - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." - continue - } - } else { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($VmDupWsp2Mv.FullName)" - $VmDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop - $VmDupWspMv = Move-Item $VmDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." - continue + if (($VmDupDstDir.CreationTimeUtc -gt $VmDupSrcDir.CreationTimeUtc) -and ($VmDupDstWsp.LastWriteTimeUtc - $VmDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { + + $VmDupDstVc = $VmDupDstDir.FullName.split("/")[5] + $VmDupDstDc = $VmDupDstDir.FullName.split("/")[6] + $VmDupDstClu = $VmDupDstDir.FullName.split("/")[7] + $VmDupSrcClu = $VmDupSrcDir.FullName.split("/")[7] + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has been moved from cluster $VmDupSrcClu to cluster $VmDupDstClu a while ago, merging metrics to the new destination if possible ..." + $VmDupWsps2Mv = Get-ChildItem -Recurse $VmDupSrcDir.FullName -Filter *.wsp + foreach ($VmDupWsp2Mv in $VmDupWsps2Mv) { + $WspRelativePath = $($VmDupWsp2Mv.FullName -split "/vm/")[1] + $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $VmDupDstVc + "/" + $VmDupDstDc + "/" + $VmDupDstClu + "/vm/" + $WspRelativePath) + if (Test-Path $DstWspFullPath) { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($VmDupWsp2Mv.FullName) to $DstWspFullPath" + $VmDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $VmDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($VmDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $VmDupWsp2MvMerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($VmDupWsp2Mv.FullName) $DstWspFullPath" + + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." + continue + } + } else { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($VmDupWsp2Mv.FullName)" + $VmDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop + $VmDupWspMv = Move-Item $VmDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." + continue + } } } + Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($VmDupSrcDir.FullName)" + try { + Remove-Item -Recurse $($VmDupSrcDir.FullName) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($VmDupSrcDir.FullName) issue ..." + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup move is too recent, has clones or has come back to its original location ..." } - Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($VmDupSrcDir.FullName)" - try { - Remove-Item -Recurse $($VmDupSrcDir.FullName) -Force -ErrorAction Stop - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($VmDupSrcDir.FullName) issue ..." - } - } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup move is too recent, has clones or has come back to its original location ..." } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated vm folders found" } - } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated vm folders found" } Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning Esx folders ..." From 6af435d353f4044bc33331c6142e4123212c0beb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 24 Feb 2023 23:40:32 +0100 Subject: [PATCH 021/201] Update ViPullStatistics.ps1 #337 --- opt/sexigraf/ViPullStatistics.ps1 | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index c8327431..3a292e25 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1008" +$ScriptVersion = "0.9.1009" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -459,14 +459,20 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { "mem.totalCapacity.average" ) - try { - $HostMultiStatsTime = Measure-Command {$HostMultiStats = MultiQueryPerfAll $($vcenter_vmhosts.moref) $HostMultiMetrics} - Write-Host "$((Get-Date).ToString("o")) [INFO] All hosts multi metrics collected in $($HostMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" - } catch { - AltAndCatchFire "ESX MultiQueryPerfAll failure" + if (($vcenter_vmhosts|Measure-Object).Count -gt 0) { + try { + $HostMultiStatsTime = Measure-Command {$HostMultiStats = MultiQueryPerfAll $($vcenter_vmhosts.moref) $HostMultiMetrics} + Write-Host "$((Get-Date).ToString("o")) [INFO] All hosts multi metrics collected in $($HostMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + } catch { + AltAndCatchFire "ESX MultiQueryPerfAll failure" + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] No ESX in vCenter $vcenter_name" } - if (($vcenter_vms|Measure-Object).Count -gt 10000) { + if (($vcenter_vms|Measure-Object).Count -eq 0) { + Write-Host "$((Get-Date).ToString("o")) [INFO] No VMs in vCenter $vcenter_name" + } elseif (($vcenter_vms|Measure-Object).Count -gt 10000) { Write-Host "$((Get-Date).ToString("o")) [INFO] 10K+ VMs mode for vCenter $vcenter_name" $VmMultiMetricsR1 = @( From 9e45d6c5543d8d9fb43e1d8c569deaa38e76075f Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Wed, 5 Apr 2023 16:40:01 +0200 Subject: [PATCH 022/201] Update ViPullStatistics.ps1 #341 --- opt/sexigraf/ViPullStatistics.ps1 | 35 ++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 846fe7b7..bc4849b7 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1007" +$ScriptVersion = "0.9.1009" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -173,7 +173,7 @@ try { Start-Transcript -Path "/var/log/sexigraf/ViPullStatistics.$($Server).log" -Append -Confirm:$false -Force -UseMinimalHeader Start-Transcript -Path "/var/log/sexigraf/ViPullStatistics.log" -Append -Confirm:$false -Force -UseMinimalHeader Write-Host "$((Get-Date).ToString("o")) [INFO] ViPullStatistics v$ScriptVersion" - if ( $vSanPull = Test-Path -Path $("/etc/cron.d/vsan_" + $Server.Replace(".","_"))) { + if ($vSanPull = Test-Path -Path $("/etc/cron.d/vsan_" + $Server.Replace(".","_"))) { Start-Transcript -Path "/var/log/sexigraf/VsanDisksPullStatistics.$($Server).log" -Append -Confirm:$false -Force -UseMinimalHeader Start-Transcript -Path "/var/log/sexigraf/VsanDisksPullStatistics.log" -Append -Confirm:$false -Force -UseMinimalHeader } @@ -272,8 +272,18 @@ try { try { if ($ServiceInstance) { Write-Host "$((Get-Date).ToString("o")) [INFO] Processing SessionManager & EventManager ..." - $SessionManager = Get-View $ServiceInstance.Content.SessionManager -Property SessionList -Server $Server # Permission to perform this operation was denied. Required privilege 'Sessions.TerminateSession' on managed object with id 'Folder-group-d1'. - $EventManager = Get-View $ServiceInstance.Content.EventManager -Property latestEvent, description -Server $Server + try { + $AuthorizationManager = Get-View $ServiceInstance.Content.AuthorizationManager -Server $Server + $UserPrivileges = $AuthorizationManager.fetchUserPrivilegeOnEntities($ServiceInstance.Content.RootFolder,$ServerConnection.User).Privileges + if ($UserPrivileges -match "Sessions.TerminateSession") { + $SessionManager = Get-View $ServiceInstance.Content.SessionManager -Property SessionList -Server $Server # Permission to perform this operation was denied. Required privilege 'Sessions.TerminateSession' on managed object with id 'Folder-group-d1'. + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] Sessions.TerminateSession privilege not detected, SessionManager skipped ..." + } + $EventManager = Get-View $ServiceInstance.Content.EventManager -Property latestEvent, description -Server $Server + } catch { + AltAndCatchFire "AuthorizationManager or SessionManager check failure" + } } else { AltAndCatchFire "ServiceInstance check failure" } @@ -1882,15 +1892,16 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name SessionList" $vcenter_session_list_h = @{} - $SessionList = $SessionManager.sessionList - - if ($SessionList) { - foreach ($vcenter_session in $SessionList) { - $vcenter_session_list_h["vi.$vcenter_name.vi.exec.sessionList.$(NameCleaner $vcenter_session.UserName)"] ++ + if ($SessionManager) { + $SessionList = $SessionManager.sessionList + if ($SessionList) { + foreach ($vcenter_session in $SessionList) { + $vcenter_session_list_h["vi.$vcenter_name.vi.exec.sessionList.$(NameCleaner $vcenter_session.UserName)"] ++ + } + $vcenter_session_list_h["vi.$vcenter_name.vi.exec.sessionCount"] = $($SessionList|Measure-Object).Count + + Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vcenter_session_list_h -DateTime $ExecStart } - $vcenter_session_list_h["vi.$vcenter_name.vi.exec.sessionCount"] = $($SessionList|Measure-Object).Count - - Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vcenter_session_list_h -DateTime $ExecStart } Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name events" From 947b37999f22ed7b4588e14707e6c3dc4b288634 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 7 Apr 2023 13:26:40 +0200 Subject: [PATCH 023/201] Create VbrConnect.ps1 #147 --- opt/veeam/VbrConnect.ps1 | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 opt/veeam/VbrConnect.ps1 diff --git a/opt/veeam/VbrConnect.ps1 b/opt/veeam/VbrConnect.ps1 new file mode 100644 index 00000000..032f98f8 --- /dev/null +++ b/opt/veeam/VbrConnect.ps1 @@ -0,0 +1,26 @@ +#!/usr/bin/pwsh -NonInteractive -NoProfile -Command +# +param([Parameter (Mandatory=$true)] [string] $server, [Parameter (Mandatory=$true)] [string] $username, [Parameter (Mandatory=$true)] [string] $password) + +# https://communities.vmware.com/t5/VMware-PowerCLI-Discussions/PowerCLI-on-Debian-Stretch-The-type-initializer-for-VMware/m-p/451739#M10646 +Set-Content -Path Env:HOME -Value '/tmp' + +if (!$(Test-Connection -TargetName $server -TcpPort 9419 -TimeoutSeconds 2)) { + Write-Host "$server is not answering at TCP:9419" + exit 1 +} + +try { + $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} + $VbrBody = @{grant_type = "password";username = $username;password = $password;refresh_token = "";code = "";use_short_term_refresh = ""} + $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + # $VbrConnect.refresh_token + Write-Host "Connected to $server" +} catch { + # Invoke-RestMethod: Unable to read data from the transport connection: Connection reset by peer. + # https://helpcenter.veeam.com/docs/backup/vbr_rest/tls_certificate.html + Write-Host "$($Error[0])" + exit 1 +} \ No newline at end of file From 73d9d71408ee9c32632c82ce417ccc56bbb5e14c Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 7 Apr 2023 13:26:45 +0200 Subject: [PATCH 024/201] Create VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 142 ++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 opt/veeam/VbrPullStatistics.ps1 diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 new file mode 100644 index 00000000..4d045a84 --- /dev/null +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -0,0 +1,142 @@ +#!/usr/bin/pwsh -Command +# +param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) + +$ScriptVersion = "0.9.1" + +$ExecStart = $(Get-Date).ToUniversalTime() +# $stopwatch = [system.diagnostics.stopwatch]::StartNew() + +$ErrorActionPreference = "SilentlyContinue" +$WarningPreference = "SilentlyContinue" +(Get-Process -Id $pid).PriorityClass = 'Idle' + +function AltAndCatchFire { + Param($ExitReason) + Write-Host "$((Get-Date).ToString("o")) [EROR] $ExitReason" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + Stop-Transcript + exit +} + +function NameCleaner { + Param($NameToClean) + $NameToClean = $NameToClean -replace "[ .]","_" + [System.Text.NormalizationForm]$NormalizationForm = "FormD" + $NameToClean = $NameToClean.Normalize($NormalizationForm) + $NameToClean = $NameToClean -replace "[^[:ascii:]]","" -replace "[^A-Za-z0-9-_]","_" + return $NameToClean.ToLower() +} + + +try { + Start-Transcript -Path "/var/log/sexigraf/VbrPullStatistics.$($Server).log" -Append -Confirm:$false -Force -UseMinimalHeader + Start-Transcript -Path "/var/log/sexigraf/VbrPullStatistics.log" -Append -Confirm:$false -Force -UseMinimalHeader + Write-Host "$((Get-Date).ToString("o")) [INFO] VbrPullStatistics v$ScriptVersion" +} catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] VbrPullStatistics logging failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + exit +} + +try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Importing Graphite PowerShell module ..." + Import-Module -Name /usr/local/share/powershell/Modules/Graphite-PowerShell-Functions/Graphite-Powershell.psm1 -Global -Force -SkipEditionCheck +} catch { + AltAndCatchFire "Powershell modules import failure" +} + +try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for another VbrPullStatistics for $Server ..." + $DupVbrPullStatisticsProcess = Get-PSHostProcessInfo|%{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}|?{$_ -match "VbrPullStatistics" -and $_ -match "$Server"} + # https://github.com/PowerShell/PowerShell/issues/13944 + if (($DupVbrPullStatisticsProcess|Measure-Object).Count -gt 1) { + $DupVbrPullStatisticsProcessId = (Get-PSHostProcessInfo|?{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '|?{$_ -match "$Server"}}).ProcessId[0] + $DupVbrPullStatisticsProcessTime = [INT32](ps -p $DupVbrPullStatisticsProcessId -o etimes).split()[-1] + if ($DupVbrPullStatisticsProcessTime -gt 300) { + Write-Host "$((Get-Date).ToString("o")) [WARN] VbrPullStatistics for $Server is already running for more than 5 minutes!" + Write-Host "$((Get-Date).ToString("o")) [WARN] Killing stunned VbrPullStatistics for $Server" + Stop-Process -Id $DupVbrPullStatisticsProcessId -Force + } else { + AltAndCatchFire "VbrPullStatistics for $Server is already running!" + } + } +} catch { + AltAndCatchFire "VbrPullStatistics process lookup failure" +} + +if ($SessionFile) { + try { + $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop + Write-Host "$((Get-Date).ToString("o")) [INFO] SessionToken found in SessionFile, attempting connection to $Server ..." + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} + $VbrServices = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/services") -Headers $VbrAuthHeaders + if ($($VbrServices.data.name)) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" + } + } catch { + Write-Host "$((Get-Date).ToString("o")) [WARN] SessionToken not found, invalid or connection failure" + Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting explicit connection ..." + } + + if (!$($VbrServices.data.name)) { + try { + $createstorexml = New-Object -TypeName XML + $createstorexml.Load($credstore) + $XPath = '//passwordEntry[server="' + $Server + '"]' + if ($(Select-XML -Xml $createstorexml -XPath $XPath)){ + $item = Select-XML -Xml $createstorexml -XPath $XPath + $CredStoreLogin = $item.Node.username + $CredStorePassword = [System.Text.Encoding]::Unicode.GetString([System.Convert]::FromBase64String($item.Node.password)) + } else { + AltAndCatchFire "No $Server entry in CredStore" + } + $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} + $VbrBody = @{grant_type = "password";username = $CredStoreLogin;password = $CredStorePassword;refresh_token = "";code = "";use_short_term_refresh = ""} + $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody + if ($($VbrServices.data.name)) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + } + } catch { + AltAndCatchFire "Explicit connection failed, check the stored credentials!" + } + } +} else { + AltAndCatchFire "No SessionFile somehow ..." +} + +if ($($VbrServices.data.name) -match "Backup") { + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)";"limit" = "9999"} + Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VBR Server $Server ..." + + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] VBR jobs states collect ..." + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] VbrJobsStates collect failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] VBR backupObjects collect ..." + $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects") -Headers $VbrAuthHeaders + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] backupObjects collect failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] VBR 5min old objectRestorePoints collect ..." + $VbrObjectRestorePoints5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/objectRestorePoints?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] objectRestorePoints collect failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + +} else { + AltAndCatchFire "VbrServices variable check failure" +} + From 9a22f0736f6facba1d447995d00be039af154b22 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 7 Apr 2023 14:44:14 +0200 Subject: [PATCH 025/201] Update ViPullStatistics.ps1 #342 --- opt/sexigraf/ViPullStatistics.ps1 | 36 +++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index ed299c2f..2c40b02c 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1009" +$ScriptVersion = "0.9.1010" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -788,25 +788,33 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } try { - $vcenter_cluster_host_power = $HostMultiStats[$PerfCounterTable["power.power.average"]][$vcenter_cluster_host.moref.value][""] - if ($vcenter_cluster_host_power -ge 0) { - $vcenter_cluster_hosts_power_usage += $vcenter_cluster_host_power - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.power", $vcenter_cluster_host_power) + if ($HostMultiStats[$PerfCounterTable["power.power.average"]][$vcenter_cluster_host.moref.value]) { + $vcenter_cluster_host_power = $HostMultiStats[$PerfCounterTable["power.power.average"]][$vcenter_cluster_host.moref.value][""] + if ($vcenter_cluster_host_power -ge 0) { + $vcenter_cluster_hosts_power_usage += $vcenter_cluster_host_power + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.power", $vcenter_cluster_host_power) + } } - $vcenter_cluster_host_cpu_totalCapacity = $HostMultiStats[$PerfCounterTable["cpu.totalCapacity.average"]][$vcenter_cluster_host.moref.value][""] - if ($vcenter_cluster_host_cpu_totalCapacity -ge 0 -and $vcenter_cluster_host.summary.quickStats.overallCpuUsage -ge 0) { - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.overallCpuUtilization", $($vcenter_cluster_host.summary.quickStats.overallCpuUsage * 100 / $vcenter_cluster_host_cpu_totalCapacity)) + if ($HostMultiStats[$PerfCounterTable["cpu.totalCapacity.average"]][$vcenter_cluster_host.moref.value]) { + $vcenter_cluster_host_cpu_totalCapacity = $HostMultiStats[$PerfCounterTable["cpu.totalCapacity.average"]][$vcenter_cluster_host.moref.value][""] + if ($vcenter_cluster_host_cpu_totalCapacity -ge 0 -and $vcenter_cluster_host.summary.quickStats.overallCpuUsage -ge 0) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.overallCpuUtilization", $($vcenter_cluster_host.summary.quickStats.overallCpuUsage * 100 / $vcenter_cluster_host_cpu_totalCapacity)) + } } - $vcenter_cluster_host_mem_totalCapacity = $HostMultiStats[$PerfCounterTable["mem.totalCapacity.average"]][$vcenter_cluster_host.moref.value][""] - if ($vcenter_cluster_host_cpu_totalCapacity -ge 0 -and $vcenter_cluster_host.summary.quickStats.overallMemoryUsage -ge 0) { - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.overallmemUtilization", $($vcenter_cluster_host.summary.quickStats.overallMemoryUsage * 100 / $vcenter_cluster_host_mem_totalCapacity)) + if ($HostMultiStats[$PerfCounterTable["mem.totalCapacity.average"]][$vcenter_cluster_host.moref.value]) { + $vcenter_cluster_host_mem_totalCapacity = $HostMultiStats[$PerfCounterTable["mem.totalCapacity.average"]][$vcenter_cluster_host.moref.value][""] + if ($vcenter_cluster_host_cpu_totalCapacity -ge 0 -and $vcenter_cluster_host.summary.quickStats.overallMemoryUsage -ge 0) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.overallmemUtilization", $($vcenter_cluster_host.summary.quickStats.overallMemoryUsage * 100 / $vcenter_cluster_host_mem_totalCapacity)) + } } - $vcenter_cluster_host_cpu_latency = $HostMultiStats[$PerfCounterTable["cpu.latency.average"]][$vcenter_cluster_host.moref.value][""] - if ($vcenter_cluster_host_cpu_latency -ge 0) { - $vcenter_cluster_hosts_cpu_latency += $vcenter_cluster_host_cpu_latency + if ($HostMultiStats[$PerfCounterTable["cpu.latency.average"]][$vcenter_cluster_host.moref.value]) { + $vcenter_cluster_host_cpu_latency = $HostMultiStats[$PerfCounterTable["cpu.latency.average"]][$vcenter_cluster_host.moref.value][""] + if ($vcenter_cluster_host_cpu_latency -ge 0) { + $vcenter_cluster_hosts_cpu_latency += $vcenter_cluster_host_cpu_latency + } } if ($vcenter_cluster_host.overallStatus.value__) { From ca90b7b2ee0f7cefa1dbd27d122708109392f372 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 7 Apr 2023 16:04:11 +0200 Subject: [PATCH 026/201] Update VbrConnect.ps1 #147 --- opt/veeam/VbrConnect.ps1 | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/opt/veeam/VbrConnect.ps1 b/opt/veeam/VbrConnect.ps1 index 032f98f8..3426d1e8 100644 --- a/opt/veeam/VbrConnect.ps1 +++ b/opt/veeam/VbrConnect.ps1 @@ -14,10 +14,15 @@ try { $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} $VbrBody = @{grant_type = "password";username = $username;password = $password;refresh_token = "";code = "";use_short_term_refresh = ""} $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody - $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" - $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName - # $VbrConnect.refresh_token - Write-Host "Connected to $server" + if ($VbrConnect.access_token) { + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + # $VbrConnect.refresh_token + Write-Host "Connected to $server" + } else { + Write-Host "Connection to $server failed!" + exit 1 + } } catch { # Invoke-RestMethod: Unable to read data from the transport connection: Connection reset by peer. # https://helpcenter.veeam.com/docs/backup/vbr_rest/tls_certificate.html From e2de91d9449bcea4ddca2327ead73d54693069b8 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 7 Apr 2023 16:04:18 +0200 Subject: [PATCH 027/201] Update VbrPullStatistics.ps1 #veeam --- opt/veeam/VbrPullStatistics.ps1 | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 4d045a84..b992977c 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1" +$ScriptVersion = "0.9.5" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -71,16 +71,18 @@ if ($SessionFile) { $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop Write-Host "$((Get-Date).ToString("o")) [INFO] SessionToken found in SessionFile, attempting connection to $Server ..." $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} - $VbrServices = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/services") -Headers $VbrAuthHeaders - if ($($VbrServices.data.name)) { + $VbrSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?limit=1") -Headers $VbrAuthHeaders + if ($($VbrSessions.data)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" + } else { + Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting explicit connection ..." } } catch { Write-Host "$((Get-Date).ToString("o")) [WARN] SessionToken not found, invalid or connection failure" Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting explicit connection ..." } - if (!$($VbrServices.data.name)) { + if (!$($VbrSessions.data)) { try { $createstorexml = New-Object -TypeName XML $createstorexml.Load($credstore) @@ -95,10 +97,19 @@ if ($SessionFile) { $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} $VbrBody = @{grant_type = "password";username = $CredStoreLogin;password = $CredStorePassword;refresh_token = "";code = "";use_short_term_refresh = ""} $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody - if ($($VbrServices.data.name)) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" - $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" - $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + if ($VbrConnect.access_token) { + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} + $VbrSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?limit=1") -Headers $VbrAuthHeaders + if ($($VbrSessions.data)) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + $SessionToken = $VbrConnect.access_token + } else { + AltAndCatchFire "VbrSessions check failed, check the user permissions!" + } + } else { + AltAndCatchFire "Explicit connection failed, check the stored credentials!" } } catch { AltAndCatchFire "Explicit connection failed, check the stored credentials!" @@ -108,8 +119,8 @@ if ($SessionFile) { AltAndCatchFire "No SessionFile somehow ..." } -if ($($VbrServices.data.name) -match "Backup") { - $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)";"limit" = "9999"} +if ($($VbrSessions.data)) { + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken";"limit" = "9999"} Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VBR Server $Server ..." try { @@ -137,6 +148,6 @@ if ($($VbrServices.data.name) -match "Backup") { } } else { - AltAndCatchFire "VbrServices variable check failure" + AltAndCatchFire "access_token check failure" } From f5c58e9458824a97926a5b234965b8814b7a9b75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sun, 9 Apr 2023 11:31:31 +0200 Subject: [PATCH 028/201] Update ViPullStatistics.ps1 #343 --- opt/sexigraf/ViPullStatistics.ps1 | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 2c40b02c..f137c2a7 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1010" +$ScriptVersion = "0.9.1011" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -1360,18 +1360,22 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { foreach ($vcenter_cluster_vsan_Ssd_uuid in $vcenter_clusters_vsan_Ssd_uuid_naa_h[$vcenter_cluster_moref].keys) { $vcenter_cluster_vsan_Ssd_host = $vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_Ssd_uuid]].name] $vcenter_cluster_vsan_Ssd_name = $vcenter_clusters_vsan_Ssd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_Ssd_uuid] - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.disk.cache.$vcenter_cluster_vsan_Ssd_name.latencyDevRead", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["latencyDevRead"]) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.disk.cache.$vcenter_cluster_vsan_Ssd_name.latencyDevWrite", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["latencyDevWrite"]) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.domcompmgr.cache.$vcenter_cluster_vsan_Ssd_name.wbFreePct", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["wbFreePct"]) + if ($vcenter_cluster_vsan_Ssd_host) { + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.disk.cache.$vcenter_cluster_vsan_Ssd_name.latencyDevRead", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["latencyDevRead"]) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.disk.cache.$vcenter_cluster_vsan_Ssd_name.latencyDevWrite", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["latencyDevWrite"]) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.domcompmgr.cache.$vcenter_cluster_vsan_Ssd_name.wbFreePct", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["wbFreePct"]) + } } foreach ($vcenter_cluster_vsan_nonSsd_uuid in $vcenter_clusters_vsan_nonSsd_uuid_naa_h[$vcenter_cluster_moref].keys) { $vcenter_cluster_vsan_nonSsd_host = $vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_nonSsd_uuid]].name] $vcenter_cluster_vsan_nonSsd_name = $vcenter_clusters_vsan_nonSsd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_nonSsd_uuid] - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.latencyDevRead", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["latencyDevRead"]) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.latencyDevWrite", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["latencyDevWrite"]) - # $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.capacityUsed", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["capacityUsed"]) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.percentUsed", $([INT64]$VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["capacityUsed"] * 100 / [INT64]$vcenter_vmhosts_vsan_disk_capa_h[$vcenter_cluster_vsan_nonSsd_uuid])) + if ($vcenter_cluster_vsan_nonSsd_host) { + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.latencyDevRead", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["latencyDevRead"]) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.latencyDevWrite", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["latencyDevWrite"]) + # $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.capacityUsed", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["capacityUsed"]) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.percentUsed", $([INT64]$VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["capacityUsed"] * 100 / [INT64]$vcenter_vmhosts_vsan_disk_capa_h[$vcenter_cluster_vsan_nonSsd_uuid])) + } } } From 66f5f989b9d66b7c7c945ea1cc6f2f1c04eaeba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sun, 9 Apr 2023 12:40:07 +0200 Subject: [PATCH 029/201] Update ViPullStatistics.ps1 #343 --- opt/sexigraf/ViPullStatistics.ps1 | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index f137c2a7..5f21439e 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1011" +$ScriptVersion = "0.9.1012" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -1350,17 +1350,20 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($vcenter_clusters_vsan_efa_h[$vcenter_cluster_moref]) { # EFA vSan cluster foreach ($PhysicalDisksHealthVsanUuid in $PhysicalDisksHealthVsanUuidHosts.keys) { $PhysicalDisksHealthVsanUuidName = NameCleaner $PhysicalDisksHealthVsanUuidObj[$PhysicalDisksHealthVsanUuid].Name - $PhysicalDisksHealthVsanUuidHost = $vcenter_vmhosts_short_h[$PhysicalDisksHealthVsanUuidHosts[$PhysicalDisksHealthVsanUuid]] - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.percentUsed", $([INT64]$PhysicalDisksHealthVsanUuidObj[$PhysicalDisksHealthVsanUuid].UsedCapacity * 100 / [INT64]$PhysicalDisksHealthVsanUuidObj[$PhysicalDisksHealthVsanUuid].Capacity)) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.latencyDevRead", $VsanPerfEntityMetric[$PhysicalDisksHealthVsanUuid]["vsan-esa-disk-scsifw"]["latencyDevRead"]) - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.latencyDevWrite", $VsanPerfEntityMetric[$PhysicalDisksHealthVsanUuid]["vsan-esa-disk-scsifw"]["latencyDevWrite"]) + if ($vcenter_vmhosts_short_h[$PhysicalDisksHealthVsanUuidHosts[$PhysicalDisksHealthVsanUuid]]) { + $PhysicalDisksHealthVsanUuidHost = $vcenter_vmhosts_short_h[$PhysicalDisksHealthVsanUuidHosts[$PhysicalDisksHealthVsanUuid]] + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.percentUsed", $([INT64]$PhysicalDisksHealthVsanUuidObj[$PhysicalDisksHealthVsanUuid].UsedCapacity * 100 / [INT64]$PhysicalDisksHealthVsanUuidObj[$PhysicalDisksHealthVsanUuid].Capacity)) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.latencyDevRead", $VsanPerfEntityMetric[$PhysicalDisksHealthVsanUuid]["vsan-esa-disk-scsifw"]["latencyDevRead"]) + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$PhysicalDisksHealthVsanUuidHost.vsan.disk.capacity.$PhysicalDisksHealthVsanUuidName.latencyDevWrite", $VsanPerfEntityMetric[$PhysicalDisksHealthVsanUuid]["vsan-esa-disk-scsifw"]["latencyDevWrite"]) + } + } } else { # Non EFA vSAN cluster foreach ($vcenter_cluster_vsan_Ssd_uuid in $vcenter_clusters_vsan_Ssd_uuid_naa_h[$vcenter_cluster_moref].keys) { - $vcenter_cluster_vsan_Ssd_host = $vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_Ssd_uuid]].name] - $vcenter_cluster_vsan_Ssd_name = $vcenter_clusters_vsan_Ssd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_Ssd_uuid] - if ($vcenter_cluster_vsan_Ssd_host) { + if ($vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_Ssd_uuid]].name]) { + $vcenter_cluster_vsan_Ssd_host = $vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_Ssd_uuid]].name] + $vcenter_cluster_vsan_Ssd_name = $vcenter_clusters_vsan_Ssd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_Ssd_uuid] $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.disk.cache.$vcenter_cluster_vsan_Ssd_name.latencyDevRead", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["latencyDevRead"]) $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.disk.cache.$vcenter_cluster_vsan_Ssd_name.latencyDevWrite", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["latencyDevWrite"]) $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_Ssd_host.vsan.domcompmgr.cache.$vcenter_cluster_vsan_Ssd_name.wbFreePct", $VsanPerfEntityMetric[$vcenter_cluster_vsan_Ssd_uuid]["cache-disk"]["wbFreePct"]) @@ -1368,9 +1371,9 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } foreach ($vcenter_cluster_vsan_nonSsd_uuid in $vcenter_clusters_vsan_nonSsd_uuid_naa_h[$vcenter_cluster_moref].keys) { - $vcenter_cluster_vsan_nonSsd_host = $vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_nonSsd_uuid]].name] - $vcenter_cluster_vsan_nonSsd_name = $vcenter_clusters_vsan_nonSsd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_nonSsd_uuid] - if ($vcenter_cluster_vsan_nonSsd_host) { + if ($vcenter_clusters_vsan_nonSsd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_nonSsd_uuid]) { + $vcenter_cluster_vsan_nonSsd_host = $vcenter_vmhosts_short_h[$vcenter_vmhosts_h[$vcenter_vmhosts_vsan_disk_moref_h[$vcenter_cluster_vsan_nonSsd_uuid]].name] + $vcenter_cluster_vsan_nonSsd_name = $vcenter_clusters_vsan_nonSsd_uuid_naa_h[$vcenter_cluster_moref][$vcenter_cluster_vsan_nonSsd_uuid] $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.latencyDevRead", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["latencyDevRead"]) $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.latencyDevWrite", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["latencyDevWrite"]) # $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_vsan_nonSsd_host.vsan.disk.capacity.$vcenter_cluster_vsan_nonSsd_name.capacityUsed", $VsanPerfEntityMetric[$vcenter_cluster_vsan_nonSsd_uuid]["capacity-disk"]["capacityUsed"]) From 4e72741318a08bb16b7e36cd8dc23b0a5d97ac6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 11 Apr 2023 13:31:55 +0200 Subject: [PATCH 030/201] Update ViPullStatistics.ps1 #342 --- opt/sexigraf/ViPullStatistics.ps1 | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 5f21439e..c59da98a 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1012" +$ScriptVersion = "0.9.1013" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -795,35 +795,55 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.power", $vcenter_cluster_host_power) } } + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats power metrics issue in cluster $vcenter_cluster_name" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + try { if ($HostMultiStats[$PerfCounterTable["cpu.totalCapacity.average"]][$vcenter_cluster_host.moref.value]) { $vcenter_cluster_host_cpu_totalCapacity = $HostMultiStats[$PerfCounterTable["cpu.totalCapacity.average"]][$vcenter_cluster_host.moref.value][""] if ($vcenter_cluster_host_cpu_totalCapacity -ge 0 -and $vcenter_cluster_host.summary.quickStats.overallCpuUsage -ge 0) { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.overallCpuUtilization", $($vcenter_cluster_host.summary.quickStats.overallCpuUsage * 100 / $vcenter_cluster_host_cpu_totalCapacity)) } } + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats cpu.totalCapacity metrics issue in cluster $vcenter_cluster_name" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + try { if ($HostMultiStats[$PerfCounterTable["mem.totalCapacity.average"]][$vcenter_cluster_host.moref.value]) { $vcenter_cluster_host_mem_totalCapacity = $HostMultiStats[$PerfCounterTable["mem.totalCapacity.average"]][$vcenter_cluster_host.moref.value][""] if ($vcenter_cluster_host_cpu_totalCapacity -ge 0 -and $vcenter_cluster_host.summary.quickStats.overallMemoryUsage -ge 0) { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.fatstats.overallmemUtilization", $($vcenter_cluster_host.summary.quickStats.overallMemoryUsage * 100 / $vcenter_cluster_host_mem_totalCapacity)) } } + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats mem.totalCapacity metrics issue in cluster $vcenter_cluster_name" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + try { if ($HostMultiStats[$PerfCounterTable["cpu.latency.average"]][$vcenter_cluster_host.moref.value]) { $vcenter_cluster_host_cpu_latency = $HostMultiStats[$PerfCounterTable["cpu.latency.average"]][$vcenter_cluster_host.moref.value][""] if ($vcenter_cluster_host_cpu_latency -ge 0) { $vcenter_cluster_hosts_cpu_latency += $vcenter_cluster_host_cpu_latency } } + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats cpu.latency metrics issue in cluster $vcenter_cluster_name" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + try { if ($vcenter_cluster_host.overallStatus.value__) { $vcenter_cluster_host_overallStatus = $vcenter_cluster_host.overallStatus.value__ } else { $vcenter_cluster_host_overallStatus = "0" } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats metrics issue in cluster $vcenter_cluster_name" + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats overallStatus metrics issue in cluster $vcenter_cluster_name" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } From 1ee33c918b94945192a6beec6bba49b7084280ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 12 Apr 2023 01:35:37 +0200 Subject: [PATCH 031/201] Update ViPullStatistics.ps1 --- opt/sexigraf/ViPullStatistics.ps1 | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index c59da98a..7b650019 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1013" +$ScriptVersion = "0.9.1014" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -842,6 +842,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } else { $vcenter_cluster_host_overallStatus = "0" } + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.overallStatus", $vcenter_cluster_host_overallStatus) } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats overallStatus metrics issue in cluster $vcenter_cluster_name" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -850,12 +851,18 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedCpuFairness", $vcenter_cluster_host.summary.quickStats.distributedCpuFairness) ### ToDo $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedMemoryFairness", $vcenter_cluster_host.summary.quickStats.distributedMemoryFairness) ### ToDo + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name Distributed Fairness quickstats issue in cluster $vcenter_cluster_name" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + + try { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.overallCpuUsage", $vcenter_cluster_host.summary.quickStats.overallCpuUsage) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.overallMemoryUsage", $vcenter_cluster_host.summary.quickStats.overallMemoryUsage) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.Uptime", $vcenter_cluster_host.summary.quickStats.uptime) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.overallStatus", $vcenter_cluster_host_overallStatus) + } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name quickstats issue in cluster $vcenter_cluster_name" + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name overall quickstats issue in cluster $vcenter_cluster_name" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } } From c3b247e1179a3112fca1ad1e9934cea5a4d2da6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 12 Apr 2023 12:16:43 +0200 Subject: [PATCH 032/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index b992977c..2f6cd896 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.5" +$ScriptVersion = "0.9.6" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -131,6 +131,16 @@ if ($($VbrSessions.data)) { Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } + if ($VbrJobsStates.data) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Building VBR jobs table ..." + $VbrJobsIdTable = @{} + foreach ($VbrJob in $VbrJobsStates.data) { + try { + $VbrJobsIdTable.add($VbrJob.id,$VbrJob) + } catch {} + } + } + try { Write-Host "$((Get-Date).ToString("o")) [INFO] VBR backupObjects collect ..." $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects") -Headers $VbrAuthHeaders @@ -141,6 +151,7 @@ if ($($VbrSessions.data)) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] VBR 5min old objectRestorePoints collect ..." + $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders $VbrObjectRestorePoints5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/objectRestorePoints?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] objectRestorePoints collect failure" From e396d6d3e6cb91fa8ec3844a884698efa4c83c77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 12 Apr 2023 12:16:45 +0200 Subject: [PATCH 033/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 67972d47..971dfdc5 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.73" +$ScriptVersion = "0.9.74" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -250,9 +250,9 @@ if ($ViServersList.count -gt 0) { $VmCluster = "" } - if ($Vm.Guest.GuestId) { + if ($Vm.Guest.GuestId.Length -gt 0) { $VmGuestId = $Vm.Guest.GuestId.replace('Guest','') - } elseif ($Vm.Config.GuestId) { + } elseif ($Vm.Config.GuestId -gt 0) { $VmGuestId = $Vm.Config.GuestId.replace('Guest','') } else { $VmGuestId = "" From a8fada8cf4f7f3fa1a7eabe739d570cee8be5057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 12 Apr 2023 12:28:47 +0200 Subject: [PATCH 034/201] Update VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 2f6cd896..770bd3d6 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -99,8 +99,8 @@ if ($SessionFile) { $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody if ($VbrConnect.access_token) { $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} - $VbrSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?limit=1") -Headers $VbrAuthHeaders - if ($($VbrSessions.data)) { + $VbrSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + if ($($VbrSessions.pagination)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName @@ -151,14 +151,19 @@ if ($($VbrSessions.data)) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] VBR 5min old objectRestorePoints collect ..." - $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + # $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders $VbrObjectRestorePoints5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/objectRestorePoints?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] objectRestorePoints collect failure" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } + Write-Host "$((Get-Date).ToString("o")) [INFO] End of VBR server $server processing ..." + } else { - AltAndCatchFire "access_token check failure" + Write-Host "$((Get-Date).ToString("o")) [INFO] Nothing to process in the last 5min on VBR server $server ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] Exit" + Stop-Transcript + exit } From af52697dfefbe6424cb1bd11aec1f7b47851477a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 14 Apr 2023 01:46:46 +0200 Subject: [PATCH 035/201] Update VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 124 +++++++++++++++++++++++--------- 1 file changed, 90 insertions(+), 34 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 770bd3d6..abce3840 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.6" +$ScriptVersion = "0.9.11" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -71,18 +71,18 @@ if ($SessionFile) { $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop Write-Host "$((Get-Date).ToString("o")) [INFO] SessionToken found in SessionFile, attempting connection to $Server ..." $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} - $VbrSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?limit=1") -Headers $VbrAuthHeaders - if ($($VbrSessions.data)) { + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states?limit=1") -Headers $VbrAuthHeaders + if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting explicit connection ..." + Write-Host "$((Get-Date).ToString("o")) [WARN] Connection failure or no job state" } } catch { Write-Host "$((Get-Date).ToString("o")) [WARN] SessionToken not found, invalid or connection failure" - Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting explicit connection ..." } - if (!$($VbrSessions.data)) { + if (!$($VbrJobsStates.data)) { + Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting explicit connection ..." try { $createstorexml = New-Object -TypeName XML $createstorexml.Load($credstore) @@ -99,14 +99,14 @@ if ($SessionFile) { $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody if ($VbrConnect.access_token) { $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} - $VbrSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders - if ($($VbrSessions.pagination)) { + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states?limit=1") -Headers $VbrAuthHeaders + if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName $SessionToken = $VbrConnect.access_token } else { - AltAndCatchFire "VbrSessions check failed, check the user permissions!" + AltAndCatchFire "Jobs States check failed, no job state or check the user permissions!" } } else { AltAndCatchFire "Explicit connection failed, check the stored credentials!" @@ -119,51 +119,107 @@ if ($SessionFile) { AltAndCatchFire "No SessionFile somehow ..." } -if ($($VbrSessions.data)) { - $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken";"limit" = "9999"} +if ($($VbrJobsStates.data)) { + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VBR Server $Server ..." try { - Write-Host "$((Get-Date).ToString("o")) [INFO] VBR jobs states collect ..." - $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders + Write-Host "$((Get-Date).ToString("o")) [INFO] VBR sessions collect ..." + $VbrRunningSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?stateFilter=working") -Headers $VbrAuthHeaders } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VbrJobsStates collect failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] Sessions collect failure" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } - if ($VbrJobsStates.data) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Building VBR jobs table ..." - $VbrJobsIdTable = @{} - foreach ($VbrJob in $VbrJobsStates.data) { - try { - $VbrJobsIdTable.add($VbrJob.id,$VbrJob) - } catch {} - } - } - - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] VBR backupObjects collect ..." - $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects") -Headers $VbrAuthHeaders - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] backupObjects collect failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + Write-Host "$((Get-Date).ToString("o")) [INFO] Building VBR sessions table ..." + $VbrRunningSessionsIdTable = @{} + foreach ($VbrSession in $VbrRunningSessions.data) { + try { + $VbrRunningSessionsIdTable.add($VbrSession.id,$VbrSession) + } catch {} } try { Write-Host "$((Get-Date).ToString("o")) [INFO] VBR 5min old objectRestorePoints collect ..." # $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders - $VbrObjectRestorePoints5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/objectRestorePoints?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + $VbrObjectRestorePoints5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/objectRestorePoints?platformNameFilter=VmWare&createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] objectRestorePoints collect failure" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } + if ($VbrObjectRestorePoints5.data.count -gt 0) { + # Write-Host "$((Get-Date).ToString("o")) [INFO] VBR backupObjects collect ..." + # $VbrBackupObjects5 = @{} + # foreach ($VbrObjectRestorePoint5 in $VbrObjectRestorePoints5.data) { + # try { + # $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&nameFilter=" + $($VbrObjectRestorePoint5.name)) -Headers $VbrAuthHeaders + # if ($VbrBackupObjects.data[0].type -eq "VM" -and !$VbrBackupObjects5[$VbrBackupObjects.data[0].name]) { + # $VbrBackupObjects5.Add($VbrBackupObjects.data[0].name,$VbrBackupObjects[0].data) + # } + # } catch { + # Write-Host "$((Get-Date).ToString("o")) [EROR] backupObjects collect failure" + # Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + # } + # } + + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] VBR backupObjects collect ..." + $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&limit=9999") -Headers $VbrAuthHeaders + if ($VbrBackupObjects.data) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Building VBR backupObjects table ..." + $VbrBackupObjectsTable = @{} + foreach ($VbrBackupObject in $VbrBackupObjects.data) { + try { + $VbrBackupObjectsTable.add($VbrBackupObject.name,$VbrBackupObject) + } catch {} + } + } + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] backupObjects collect failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + + $VbrObjectRestorePoints5SessionsId = $VbrObjectRestorePoints5.data.backupId|Select-Object -Unique + + Write-Host "$((Get-Date).ToString("o")) [INFO] VBR Sessions collect ..." + $VbrSessions5 = @{} + foreach ($VbrObjectRestorePoints5SessionId in $VbrObjectRestorePoints5SessionsId) { + try { + $VbrObjectRestorePoints5Session = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backups/" + $VbrObjectRestorePoints5SessionId) -Headers $VbrAuthHeaders + if ($VbrObjectRestorePoints5Session) { + $VbrSessions5.Add($VbrObjectRestorePoints5SessionId,$VbrObjectRestorePoints5Session) + } + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] backupObjects collect failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + } + + $VbrObjectRestorePointTable = @{} + + $vbrserver_name = NameCleaner $Server + + foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { + $vcenter_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].path.split("\")[0] + $vm_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name + $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name + $VbrObjectRestorePointTable["veeam.vi.$vcenter_name.vm.$vm_name"] ++ + $VbrObjectRestorePointTable["veeam.vbr.$vbrserver_name.job.$job_name"] ++ + } + + + $VbrObjectRestorePointTable["vi.$vbrserver_name.vi.exec.duration"] = $($(Get-Date).ToUniversalTime() - $ExecStart).TotalSeconds + Write-Host "$((Get-Date).ToString("o")) [INFO] Sending veeam data to Graphite for VBR server $Server ..." + Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrObjectRestorePointTable -DateTime $ExecStart + + } + Write-Host "$((Get-Date).ToString("o")) [INFO] End of VBR server $server processing ..." } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] Nothing to process in the last 5min on VBR server $server ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] No Running Jobs on VBR server $server ..." Write-Host "$((Get-Date).ToString("o")) [INFO] Exit" Stop-Transcript exit -} - +} \ No newline at end of file From 80262f6e301b962a93d2641e66e5e6de994942d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 14 Apr 2023 15:10:11 +0200 Subject: [PATCH 036/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 43 +++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index abce3840..35704d36 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.11" +$ScriptVersion = "0.9.13" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -125,7 +125,7 @@ if ($($VbrJobsStates.data)) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] VBR sessions collect ..." - $VbrRunningSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?stateFilter=working") -Headers $VbrAuthHeaders + $VbrRunningSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?endedAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] Sessions collect failure" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -136,6 +136,7 @@ if ($($VbrJobsStates.data)) { foreach ($VbrSession in $VbrRunningSessions.data) { try { $VbrRunningSessionsIdTable.add($VbrSession.id,$VbrSession) + Write-Host $VbrSession #TODO clean } catch {} } @@ -151,7 +152,7 @@ if ($($VbrJobsStates.data)) { if ($VbrObjectRestorePoints5.data.count -gt 0) { # Write-Host "$((Get-Date).ToString("o")) [INFO] VBR backupObjects collect ..." # $VbrBackupObjects5 = @{} - # foreach ($VbrObjectRestorePoint5 in $VbrObjectRestorePoints5.data) { + # foreach ($VbrObjectRestorePoint5 in $VbrObjectRestorePoints5.data) { # Too Slow !!! # try { # $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&nameFilter=" + $($VbrObjectRestorePoint5.name)) -Headers $VbrAuthHeaders # if ($VbrBackupObjects.data[0].type -eq "VM" -and !$VbrBackupObjects5[$VbrBackupObjects.data[0].name]) { @@ -197,15 +198,43 @@ if ($($VbrJobsStates.data)) { } $VbrObjectRestorePointTable = @{} - $vbrserver_name = NameCleaner $Server foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { - $vcenter_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].path.split("\")[0] - $vm_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name - $VbrObjectRestorePointTable["veeam.vi.$vcenter_name.vm.$vm_name"] ++ $VbrObjectRestorePointTable["veeam.vbr.$vbrserver_name.job.$job_name"] ++ + + if (Test-Path /mnt/wfs/inventory/ViVmInventory.csv) { + try { + $ViVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop) + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] ViVmInventory import issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + + if ($ViVmInventory) { + $ViVmInventoryTable = @{} + foreach ($ViVm in $ViVmInventory) { + try { + $ViVmInventoryTable.Add($ViVm.vm,$ViVm) + } catch {} + } + } else { + Write-Host "$((Get-Date).ToString("o")) [EROR] ViVmInventory import issue" + } + + $vcenter_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].path.split("\")[0] + $vm_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name + + if ($ViVmInventoryTable[$vm_name]) { + $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].Cluster + $VbrObjectRestorePointTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.objectRestorePoints"] ++ + $VbrObjectRestorePointTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount + } + + } else { + Write-Host "$((Get-Date).ToString("o")) [EROR] No ViVmInventory" + } } From 7d346ec996d6dc0a566c35caea6bab3f690d4ffb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 14 Apr 2023 19:22:57 +0200 Subject: [PATCH 037/201] Update VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 57 ++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 35704d36..6fdb1008 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.13" +$ScriptVersion = "0.9.15" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -121,25 +121,10 @@ if ($SessionFile) { if ($($VbrJobsStates.data)) { $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} + $VbrObjectRestorePointTable = @{} + $vbrserver_name = NameCleaner $Server Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VBR Server $Server ..." - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] VBR sessions collect ..." - $VbrRunningSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?endedAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Sessions collect failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" - } - - Write-Host "$((Get-Date).ToString("o")) [INFO] Building VBR sessions table ..." - $VbrRunningSessionsIdTable = @{} - foreach ($VbrSession in $VbrRunningSessions.data) { - try { - $VbrRunningSessionsIdTable.add($VbrSession.id,$VbrSession) - Write-Host $VbrSession #TODO clean - } catch {} - } - try { Write-Host "$((Get-Date).ToString("o")) [INFO] VBR 5min old objectRestorePoints collect ..." # $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders @@ -197,12 +182,9 @@ if ($($VbrJobsStates.data)) { } } - $VbrObjectRestorePointTable = @{} - $vbrserver_name = NameCleaner $Server - foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name - $VbrObjectRestorePointTable["veeam.vbr.$vbrserver_name.job.$job_name"] ++ + $VbrObjectRestorePointTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ if (Test-Path /mnt/wfs/inventory/ViVmInventory.csv) { try { @@ -236,14 +218,37 @@ if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [EROR] No ViVmInventory" } } + } - - $VbrObjectRestorePointTable["vi.$vbrserver_name.vi.exec.duration"] = $($(Get-Date).ToUniversalTime() - $ExecStart).TotalSeconds - Write-Host "$((Get-Date).ToString("o")) [INFO] Sending veeam data to Graphite for VBR server $Server ..." - Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrObjectRestorePointTable -DateTime $ExecStart + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] VBR ended sessions collect ..." + $VbrEndedSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?endedAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Sessions collect failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + if ($VbrEndedSessions.data) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Processing VBR ended sessions ..." + foreach ($VbrEndedSession in $VbrEndedSessions.data) { + $job_name = NameCleaner $VbrEndedSession.name + if ($VbrEndedSession.result.result -eq "Success") { + $VbrEndedSessionResult = 0 + } elseif ($VbrEndedSession.result.result -eq "Warning") { + $VbrEndedSessionResult = 1 + } elseif ($VbrEndedSession.result.result -eq "Failed") { + $VbrEndedSessionResult = 2 + } else { + $VbrEndedSessionResult = 3 + } + $VbrObjectRestorePointTable["veeam.vbr.$vbrserver_name.job.$job_name.result"] = $VbrEndedSessionResult + } } + $VbrObjectRestorePointTable["vi.$vbrserver_name.vi.exec.duration"] = $($(Get-Date).ToUniversalTime() - $ExecStart).TotalSeconds #TODO ? + Write-Host "$((Get-Date).ToString("o")) [INFO] Sending veeam data to Graphite for VBR server $Server ..." + Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrObjectRestorePointTable -DateTime $ExecStart + Write-Host "$((Get-Date).ToString("o")) [INFO] End of VBR server $server processing ..." } else { From e7a9b6aa1c15bfacab28374d017868c15ffb326e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 14 Apr 2023 20:21:43 +0200 Subject: [PATCH 038/201] Update VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 6fdb1008..df4e1024 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.15" +$ScriptVersion = "0.9.16" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -71,7 +71,7 @@ if ($SessionFile) { $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop Write-Host "$((Get-Date).ToString("o")) [INFO] SessionToken found in SessionFile, attempting connection to $Server ..." $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} - $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states?limit=1") -Headers $VbrAuthHeaders + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" } else { @@ -99,7 +99,7 @@ if ($SessionFile) { $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody if ($VbrConnect.access_token) { $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} - $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states?limit=1") -Headers $VbrAuthHeaders + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" @@ -125,6 +125,20 @@ if ($($VbrJobsStates.data)) { $vbrserver_name = NameCleaner $Server Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VBR Server $Server ..." + foreach ($VbrJobState in $VbrJobsStates.data) { + $job_name = NameCleaner $VbrJobState.name + if ($VbrJobState.status -eq "running") { + $VbrJobStateStatus = 0 + } elseif ($VbrJobState.status -eq "inactive") { + $VbrJobStateStatus = 1 + } elseif ($VbrJobState.status -eq "disabled") { + $VbrJobStateStatus = 2 + } else { + $VbrJobStateStatus = 3 + } + $VbrObjectRestorePointTable["veeam.vbr.$vbrserver_name.job.$job_name.status"] = $VbrJobStateStatus + } + try { Write-Host "$((Get-Date).ToString("o")) [INFO] VBR 5min old objectRestorePoints collect ..." # $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders From cfe0613f36f1798d877e358382aefe1410e9882e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Mon, 17 Apr 2023 01:23:25 +0200 Subject: [PATCH 039/201] Update storage-aggregation.conf #147 --- opt/graphite/conf/storage-aggregation.conf | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/opt/graphite/conf/storage-aggregation.conf b/opt/graphite/conf/storage-aggregation.conf index c2b18616..6dc1859b 100644 --- a/opt/graphite/conf/storage-aggregation.conf +++ b/opt/graphite/conf/storage-aggregation.conf @@ -41,4 +41,19 @@ aggregationMethod = sum [errorsTx] pattern = .\.errorsTx. xFilesFactor = 0.0 -aggregationMethod = sum \ No newline at end of file +aggregationMethod = sum + +[veeamObjectRestorePoints] +pattern = ^veeam.*\.objectRestorePoints +xFilesFactor = 0.0 +aggregationMethod = last + +[veeamRestorePointsCount] +pattern = ^veeam.*\.restorePointsCount +xFilesFactor = 0.0 +aggregationMethod = sum + +[veeamResult] +pattern = ^veeam.*\.result +xFilesFactor = 0.0 +aggregationMethod = last \ No newline at end of file From a38d8f6359241b5b1d7d8fdbfddd6767b3c0ac76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Mon, 17 Apr 2023 01:23:33 +0200 Subject: [PATCH 040/201] Update storage-schemas.conf #147 --- opt/graphite/conf/storage-schemas.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opt/graphite/conf/storage-schemas.conf b/opt/graphite/conf/storage-schemas.conf index 98024f30..fa4721c1 100644 --- a/opt/graphite/conf/storage-schemas.conf +++ b/opt/graphite/conf/storage-schemas.conf @@ -50,6 +50,10 @@ retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y pattern = ^vsan\. retentions = 5m:12h,10m:24h,20m:48h,80m:7d,320m:30d,640m:90d +[veeam] +pattern = ^veeam\. +retentions = 5m:7d,10m:30d,15m:90d,30m:120d,120m:1y + [anything_else] pattern = .* retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y From 28f4cca0c2301b8af0a3d61dd517cba97dc1b322 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 18 Apr 2023 14:32:05 +0200 Subject: [PATCH 041/201] Update storage-aggregation.conf #258 --- opt/graphite/conf/storage-aggregation.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/opt/graphite/conf/storage-aggregation.conf b/opt/graphite/conf/storage-aggregation.conf index 6dc1859b..5ab5c9de 100644 --- a/opt/graphite/conf/storage-aggregation.conf +++ b/opt/graphite/conf/storage-aggregation.conf @@ -43,6 +43,11 @@ pattern = .\.errorsTx. xFilesFactor = 0.0 aggregationMethod = sum +[PathState] +pattern = .\.superstats\.PathState\.* +xFilesFactor = 0.0 +aggregationMethod = last + [veeamObjectRestorePoints] pattern = ^veeam.*\.objectRestorePoints xFilesFactor = 0.0 From 820bb4228fc5ffc96296133f1bc5c8079dddacbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 18 Apr 2023 14:33:52 +0200 Subject: [PATCH 042/201] Update storage-aggregation.conf --- opt/graphite/conf/storage-aggregation.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/opt/graphite/conf/storage-aggregation.conf b/opt/graphite/conf/storage-aggregation.conf index 5ab5c9de..caa301df 100644 --- a/opt/graphite/conf/storage-aggregation.conf +++ b/opt/graphite/conf/storage-aggregation.conf @@ -48,6 +48,11 @@ pattern = .\.superstats\.PathState\.* xFilesFactor = 0.0 aggregationMethod = last +[overallStatus] +pattern = .\.quickstats\.overallStatus\.* +xFilesFactor = 0.0 +aggregationMethod = last + [veeamObjectRestorePoints] pattern = ^veeam.*\.objectRestorePoints xFilesFactor = 0.0 From caef1f3b48ee800e9219eee5bf3e57b50113df95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 19 Apr 2023 23:02:57 +0200 Subject: [PATCH 043/201] Update storage-aggregation.conf #147 --- opt/graphite/conf/storage-aggregation.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/opt/graphite/conf/storage-aggregation.conf b/opt/graphite/conf/storage-aggregation.conf index caa301df..ebc88992 100644 --- a/opt/graphite/conf/storage-aggregation.conf +++ b/opt/graphite/conf/storage-aggregation.conf @@ -66,4 +66,9 @@ aggregationMethod = sum [veeamResult] pattern = ^veeam.*\.result xFilesFactor = 0.0 +aggregationMethod = last + +[veeamStatus] +pattern = ^veeam.*\.status +xFilesFactor = 0.0 aggregationMethod = last \ No newline at end of file From 68d217980e93e6a77fd0eb155a5dc3f7e1b1e907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 20 Apr 2023 00:53:08 +0200 Subject: [PATCH 044/201] Update ViPullStatistics.ps1 #341 --- opt/sexigraf/ViPullStatistics.ps1 | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 7b650019..5a1813d2 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1014" +$ScriptVersion = "0.9.1017" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -274,8 +274,11 @@ try { Write-Host "$((Get-Date).ToString("o")) [INFO] Processing SessionManager & EventManager ..." try { $AuthorizationManager = Get-View $ServiceInstance.Content.AuthorizationManager -Server $Server - $UserPrivileges = $AuthorizationManager.fetchUserPrivilegeOnEntities($ServiceInstance.Content.RootFolder,$ServerConnection.User).Privileges - if ($UserPrivileges -match "Sessions.TerminateSession") { + $group_d1 = Get-View $ServiceInstance.Content.RootFolder + $UserRoleId = $($group_d1.Permission|?{$_.Principal -eq $ServerConnection.User}).RoleId + $UserRole = $AuthorizationManager.RoleList|?{$_.RoleId -eq $UserRoleId} + # $UserTerminateSessionCheck = $AuthorizationManager.HasUserPrivilegeOnEntities($ServiceInstance.Content.RootFolder,$ServerConnection.User,"Sessions.TerminateSession") + if ($UserRole.Privilege -match "Sessions.TerminateSession") { $SessionManager = Get-View $ServiceInstance.Content.SessionManager -Property SessionList -Server $Server # Permission to perform this operation was denied. Required privilege 'Sessions.TerminateSession' on managed object with id 'Folder-group-d1'. } else { Write-Host "$((Get-Date).ToString("o")) [INFO] Sessions.TerminateSession privilege not detected, SessionManager skipped ..." @@ -1954,10 +1957,9 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name SessionList" - - $vcenter_session_list_h = @{} if ($SessionManager) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name SessionList" + $vcenter_session_list_h = @{} $SessionList = $SessionManager.sessionList if ($SessionList) { foreach ($vcenter_session in $SessionList) { From ec04d17e2d99abb097a896e5b9ee5266f8c6fb58 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 27 Apr 2023 19:05:42 +0200 Subject: [PATCH 045/201] Update VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 40 +++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index df4e1024..ff5b06f0 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.16" +$ScriptVersion = "0.9.17" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -121,7 +121,7 @@ if ($SessionFile) { if ($($VbrJobsStates.data)) { $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} - $VbrObjectRestorePointTable = @{} + $VbrDataTable = @{} $vbrserver_name = NameCleaner $Server Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VBR Server $Server ..." @@ -136,7 +136,29 @@ if ($($VbrJobsStates.data)) { } else { $VbrJobStateStatus = 3 } - $VbrObjectRestorePointTable["veeam.vbr.$vbrserver_name.job.$job_name.status"] = $VbrJobStateStatus + $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.status"] = $VbrJobStateStatus + } + + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] VBR repositories states collect ..." + $VbrRepositoriesStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupInfrastructure/repositories/states") -Headers $VbrAuthHeaders + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] repositories states collect failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + + if ($VbrRepositoriesStates.data.count -gt 0) { + foreach ($VbrRepository in $VbrRepositoriesStates.data) { + $VbrRepositoryName = NameCleaner $VbrRepository.name + + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.capacityGB"] = $VbrRepository.capacityGB + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.freeGB"] = $VbrRepository.freeGB + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.usedSpaceGB"] = $VbrRepository.usedSpaceGB + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.freePct"] = $($VbrRepository.freeGB * 100 / $VbrRepository.capacityGB) + + } + } else { + Write-Host "$((Get-Date).ToString("o")) [WARN] No repositories ?!" } try { @@ -198,7 +220,7 @@ if ($($VbrJobsStates.data)) { foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name - $VbrObjectRestorePointTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ + $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ if (Test-Path /mnt/wfs/inventory/ViVmInventory.csv) { try { @@ -224,8 +246,8 @@ if ($($VbrJobsStates.data)) { if ($ViVmInventoryTable[$vm_name]) { $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].Cluster - $VbrObjectRestorePointTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.objectRestorePoints"] ++ - $VbrObjectRestorePointTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount + $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.objectRestorePoints"] ++ + $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount } } else { @@ -255,13 +277,13 @@ if ($($VbrJobsStates.data)) { } else { $VbrEndedSessionResult = 3 } - $VbrObjectRestorePointTable["veeam.vbr.$vbrserver_name.job.$job_name.result"] = $VbrEndedSessionResult + $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.result"] = $VbrEndedSessionResult } } - $VbrObjectRestorePointTable["vi.$vbrserver_name.vi.exec.duration"] = $($(Get-Date).ToUniversalTime() - $ExecStart).TotalSeconds #TODO ? + $VbrDataTable["vi.$vbrserver_name.vi.exec.duration"] = $($(Get-Date).ToUniversalTime() - $ExecStart).TotalSeconds #TODO ? Write-Host "$((Get-Date).ToString("o")) [INFO] Sending veeam data to Graphite for VBR server $Server ..." - Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrObjectRestorePointTable -DateTime $ExecStart + Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrDataTable -DateTime $ExecStart Write-Host "$((Get-Date).ToString("o")) [INFO] End of VBR server $server processing ..." From 1fcd577d0a63367e6ecc1cf98f00ce5a5734d1fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 29 Apr 2023 22:50:21 +0200 Subject: [PATCH 046/201] Update VbrConnect.ps1 --- opt/veeam/VbrConnect.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opt/veeam/VbrConnect.ps1 b/opt/veeam/VbrConnect.ps1 index 3426d1e8..8feaa88a 100644 --- a/opt/veeam/VbrConnect.ps1 +++ b/opt/veeam/VbrConnect.ps1 @@ -16,8 +16,9 @@ try { $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody if ($VbrConnect.access_token) { $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $SessionRefresh = "vbr_" + $server.Replace(".","_") + ".dat" $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName - # $VbrConnect.refresh_token + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefresh Write-Host "Connected to $server" } else { Write-Host "Connection to $server failed!" From a9cd5327e6d8a35e8f579aa1c92d6219173f4fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 29 Apr 2023 22:50:29 +0200 Subject: [PATCH 047/201] Update VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 47 ++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index ff5b06f0..933356d1 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.17" +$ScriptVersion = "0.9.18" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -74,12 +74,51 @@ if ($SessionFile) { $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" + Write-Host "$((Get-Date).ToString("o")) [INFO] Refreshing Token" + $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" + $SessionRefresh = Get-Content -Path /tmp/$SessionRefreshPath -ErrorAction Stop + $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} + $VbrBody = @{grant_type = "refresh_token";username = "";password = "";refresh_token = $SessionRefresh;code = $SessionToken;use_short_term_refresh = ""} + $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody + if ($VbrConnect.access_token) { + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $SessionRefresh = "vbr_" + $server.Replace(".","_") + ".dat" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefresh + $SessionToken = $VbrConnect.access_token + } else { + Write-Host "$((Get-Date).ToString("o")) [WARN] Token refresh failed!" + } } else { Write-Host "$((Get-Date).ToString("o")) [WARN] Connection failure or no job state" } } catch { Write-Host "$((Get-Date).ToString("o")) [WARN] SessionToken not found, invalid or connection failure" } + + if (!$($VbrJobsStates.data)) { + Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting token refresh ..." + try { + $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" + $SessionRefresh = Get-Content -Path /tmp/$SessionRefreshPath -ErrorAction Stop + $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} + $VbrBody = @{grant_type = "refresh_token";username = "";password = "";refresh_token = $SessionRefresh;code = $SessionToken;use_short_term_refresh = ""} + $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody + if ($VbrConnect.access_token) { + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $SessionRefresh = "vbr_" + $server.Replace(".","_") + ".dat" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefresh + $SessionToken = $VbrConnect.access_token + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders + } else { + Write-Host "$((Get-Date).ToString("o")) [WARN] Token refresh failed!" + } + } catch { + Write-Host "$((Get-Date).ToString("o")) [WARN] Token refresh issue!" + } + } if (!$($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting explicit connection ..." @@ -103,7 +142,9 @@ if ($SessionFile) { if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $SessionRefresh = "vbr_" + $server.Replace(".","_") + ".dat" $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefresh $SessionToken = $VbrConnect.access_token } else { AltAndCatchFire "Jobs States check failed, no job state or check the user permissions!" @@ -147,7 +188,7 @@ if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } - if ($VbrRepositoriesStates.data.count -gt 0) { + if ($VbrRepositoriesStates.data) { foreach ($VbrRepository in $VbrRepositoriesStates.data) { $VbrRepositoryName = NameCleaner $VbrRepository.name @@ -170,7 +211,7 @@ if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } - if ($VbrObjectRestorePoints5.data.count -gt 0) { + if ($VbrObjectRestorePoints5.data) { # Write-Host "$((Get-Date).ToString("o")) [INFO] VBR backupObjects collect ..." # $VbrBackupObjects5 = @{} # foreach ($VbrObjectRestorePoint5 in $VbrObjectRestorePoints5.data) { # Too Slow !!! From f5188cf765e685854437346a833bc0c44442b263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 6 May 2023 01:13:57 +0200 Subject: [PATCH 048/201] Update VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 933356d1..bf2beac2 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.18" +$ScriptVersion = "0.9.19" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -74,21 +74,6 @@ if ($SessionFile) { $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" - Write-Host "$((Get-Date).ToString("o")) [INFO] Refreshing Token" - $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" - $SessionRefresh = Get-Content -Path /tmp/$SessionRefreshPath -ErrorAction Stop - $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} - $VbrBody = @{grant_type = "refresh_token";username = "";password = "";refresh_token = $SessionRefresh;code = $SessionToken;use_short_term_refresh = ""} - $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody - if ($VbrConnect.access_token) { - $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" - $SessionRefresh = "vbr_" + $server.Replace(".","_") + ".dat" - $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName - $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefresh - $SessionToken = $VbrConnect.access_token - } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] Token refresh failed!" - } } else { Write-Host "$((Get-Date).ToString("o")) [WARN] Connection failure or no job state" } @@ -102,16 +87,18 @@ if ($SessionFile) { $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" $SessionRefresh = Get-Content -Path /tmp/$SessionRefreshPath -ErrorAction Stop $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} - $VbrBody = @{grant_type = "refresh_token";username = "";password = "";refresh_token = $SessionRefresh;code = $SessionToken;use_short_term_refresh = ""} + $VbrBody = @{grant_type = "refresh_token";username = "";password = "";refresh_token = $SessionRefresh;code = "";use_short_term_refresh = ""} $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody if ($VbrConnect.access_token) { $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" - $SessionRefresh = "vbr_" + $server.Replace(".","_") + ".dat" $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName - $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefresh + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefreshPath $SessionToken = $VbrConnect.access_token $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders + if (!$($VbrJobsStates.data)) { + Write-Host "$((Get-Date).ToString("o")) [WARN] Known issue on v11!" + } } else { Write-Host "$((Get-Date).ToString("o")) [WARN] Token refresh failed!" } @@ -142,9 +129,9 @@ if ($SessionFile) { if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" - $SessionRefresh = "vbr_" + $server.Replace(".","_") + ".dat" + $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName - $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefresh + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefreshPath $SessionToken = $VbrConnect.access_token } else { AltAndCatchFire "Jobs States check failed, no job state or check the user permissions!" @@ -200,6 +187,9 @@ if ($($VbrJobsStates.data)) { } } else { Write-Host "$((Get-Date).ToString("o")) [WARN] No repositories ?!" + # errorCode message resourceId + # --------- ------- ---------- + # AccessDenied Forbidden } try { From 634dbab38fff551cf521bfdc429a283216d7a69b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 6 May 2023 01:21:31 +0200 Subject: [PATCH 049/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index bf2beac2..5ad75683 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -98,6 +98,7 @@ if ($SessionFile) { $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders if (!$($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [WARN] Known issue on v11!" + # https://forums.veeam.com/restful-api-f30/how-to-handle-the-refresh-token-t74916.html } } else { Write-Host "$((Get-Date).ToString("o")) [WARN] Token refresh failed!" From 1a24a8f7ea9cb8df6bb315e0a7a98b6d3022586e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 6 May 2023 02:16:44 +0200 Subject: [PATCH 050/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 5ad75683..c8fd2b4c 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -97,6 +97,7 @@ if ($SessionFile) { $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders if (!$($VbrJobsStates.data)) { + Write-Host "$((Get-Date).ToString("o")) [WARN] Token refresh failed!" Write-Host "$((Get-Date).ToString("o")) [WARN] Known issue on v11!" # https://forums.veeam.com/restful-api-f30/how-to-handle-the-refresh-token-t74916.html } From 6469b3f899df8aa374c72e2fd4ad00176cc67411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 10 May 2023 10:38:21 +0200 Subject: [PATCH 051/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index c8fd2b4c..0391cb3c 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.19" +$ScriptVersion = "0.9.20" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -68,17 +68,22 @@ try { if ($SessionFile) { try { - $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop - Write-Host "$((Get-Date).ToString("o")) [INFO] SessionToken found in SessionFile, attempting connection to $Server ..." - $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} - $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders - if ($($VbrJobsStates.data)) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" + $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" + if ([DateTime]$(Get-Content -Path /tmp/$SessionSecretExpiration) -gt $ExecStart.AddMinutes(5)) { + $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop + Write-Host "$((Get-Date).ToString("o")) [INFO] SessionToken found in SessionFile, attempting connection to $Server ..." + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders + if ($($VbrJobsStates.data)) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" + } else { + Write-Host "$((Get-Date).ToString("o")) [WARN] Connection failure or no job state" + } } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] Connection failure or no job state" + Write-Host "$((Get-Date).ToString("o")) [WARN] Token is about to expire ..." } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] SessionToken not found, invalid or connection failure" + Write-Host "$((Get-Date).ToString("o")) [WARN] SessionToken or SessionSecretExpiration not found, invalid or connection failure" } if (!$($VbrJobsStates.data)) { @@ -91,8 +96,10 @@ if ($SessionFile) { $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody if ($VbrConnect.access_token) { $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefreshPath + $VbrConnect.".expires".ToUniversalTime().tostring() | Out-File -FilePath /tmp/$SessionSecretExpiration $SessionToken = $VbrConnect.access_token $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders @@ -132,8 +139,10 @@ if ($SessionFile) { Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" + $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName - $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefreshPath + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefreshPath + $VbrConnect.".expires".ToUniversalTime().tostring() | Out-File -FilePath /tmp/$SessionSecretExpiration $SessionToken = $VbrConnect.access_token } else { AltAndCatchFire "Jobs States check failed, no job state or check the user permissions!" From 51d287da74390862a9aab303f56da8e45ae6e58b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 10 May 2023 11:21:05 +0200 Subject: [PATCH 052/201] Update VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 0391cb3c..e57329a4 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -203,6 +203,14 @@ if ($($VbrJobsStates.data)) { # AccessDenied Forbidden } + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] VBR ScaleOutRepositories collect ..." + $VbrScaleOutRepositories = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupInfrastructure/scaleOutRepositories") -Headers $VbrAuthHeaders + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] ScaleOutRepositories collect failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + try { Write-Host "$((Get-Date).ToString("o")) [INFO] VBR 5min old objectRestorePoints collect ..." # $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders @@ -215,7 +223,7 @@ if ($($VbrJobsStates.data)) { if ($VbrObjectRestorePoints5.data) { # Write-Host "$((Get-Date).ToString("o")) [INFO] VBR backupObjects collect ..." # $VbrBackupObjects5 = @{} - # foreach ($VbrObjectRestorePoint5 in $VbrObjectRestorePoints5.data) { # Too Slow !!! + # foreach ($VbrObjectRestorePoint5 in $VbrObjectRestorePoints5.data) { # try { # $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&nameFilter=" + $($VbrObjectRestorePoint5.name)) -Headers $VbrAuthHeaders # if ($VbrBackupObjects.data[0].type -eq "VM" -and !$VbrBackupObjects5[$VbrBackupObjects.data[0].name]) { @@ -225,11 +233,11 @@ if ($($VbrJobsStates.data)) { # Write-Host "$((Get-Date).ToString("o")) [EROR] backupObjects collect failure" # Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" # } - # } + # } # Too Slow !!! try { Write-Host "$((Get-Date).ToString("o")) [INFO] VBR backupObjects collect ..." - $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&limit=9999") -Headers $VbrAuthHeaders + $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&limit=9999") -Headers $VbrAuthHeaders # TODO limit=9999+ ? if ($VbrBackupObjects.data) { Write-Host "$((Get-Date).ToString("o")) [INFO] Building VBR backupObjects table ..." $VbrBackupObjectsTable = @{} From 0edeeb70fd103b7f05850a051384279638209265 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Wed, 10 May 2023 20:31:04 +0200 Subject: [PATCH 053/201] Update VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index e57329a4..d3491a54 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.20" +$ScriptVersion = "0.9.21" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -187,6 +187,7 @@ if ($($VbrJobsStates.data)) { } if ($VbrRepositoriesStates.data) { + $VbrRepositoryTable = @{} foreach ($VbrRepository in $VbrRepositoriesStates.data) { $VbrRepositoryName = NameCleaner $VbrRepository.name @@ -195,6 +196,9 @@ if ($($VbrJobsStates.data)) { $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.usedSpaceGB"] = $VbrRepository.usedSpaceGB $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.freePct"] = $($VbrRepository.freeGB * 100 / $VbrRepository.capacityGB) + try { + $VbrRepositoryTable.add($VbrRepository.id,$VbrRepository) + } catch {} } } else { Write-Host "$((Get-Date).ToString("o")) [WARN] No repositories ?!" From 19cae6a217b913fc1302274301788e1e3ca9f878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 11 May 2023 17:46:41 +0200 Subject: [PATCH 054/201] Update VbrConnect.ps1 #147 --- opt/veeam/VbrConnect.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opt/veeam/VbrConnect.ps1 b/opt/veeam/VbrConnect.ps1 index 8feaa88a..c5a7baa6 100644 --- a/opt/veeam/VbrConnect.ps1 +++ b/opt/veeam/VbrConnect.ps1 @@ -6,7 +6,7 @@ param([Parameter (Mandatory=$true)] [string] $server, [Parameter (Mandatory=$tru Set-Content -Path Env:HOME -Value '/tmp' if (!$(Test-Connection -TargetName $server -TcpPort 9419 -TimeoutSeconds 2)) { - Write-Host "$server is not answering at TCP:9419" + Write-Host "$server is not answering at TCP:9419 check if VeeamBackupRESTSvc is running" exit 1 } From baf3a45ae0fd8ce388298eb723bbefdd21765aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 11 May 2023 23:38:48 +0200 Subject: [PATCH 055/201] Update VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 43 ++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index d3491a54..953afe7a 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.21" +$ScriptVersion = "0.9.22" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -80,7 +80,7 @@ if ($SessionFile) { Write-Host "$((Get-Date).ToString("o")) [WARN] Connection failure or no job state" } } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] Token is about to expire ..." + Write-Host "$((Get-Date).ToString("o")) [WARN] Token has expired or is about to ..." } } catch { Write-Host "$((Get-Date).ToString("o")) [WARN] SessionToken or SessionSecretExpiration not found, invalid or connection failure" @@ -187,6 +187,7 @@ if ($($VbrJobsStates.data)) { } if ($VbrRepositoriesStates.data) { + Write-Host "$((Get-Date).ToString("o")) [INFO] VBR repositories processing ..." $VbrRepositoryTable = @{} foreach ($VbrRepository in $VbrRepositoriesStates.data) { $VbrRepositoryName = NameCleaner $VbrRepository.name @@ -194,7 +195,7 @@ if ($($VbrJobsStates.data)) { $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.capacityGB"] = $VbrRepository.capacityGB $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.freeGB"] = $VbrRepository.freeGB $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.usedSpaceGB"] = $VbrRepository.usedSpaceGB - $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.freePct"] = $($VbrRepository.freeGB * 100 / $VbrRepository.capacityGB) + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.RealUsedPct"] = $($VbrRepository.usedSpaceGB * 100 / $($VbrRepository.usedSpaceGB + $VbrRepository.freeGB )) try { $VbrRepositoryTable.add($VbrRepository.id,$VbrRepository) @@ -213,7 +214,41 @@ if ($($VbrJobsStates.data)) { } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] ScaleOutRepositories collect failure" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" - } + } + + if ($VbrScaleOutRepositories.data) { + Write-Host "$((Get-Date).ToString("o")) [INFO] SOBR performanceTier processing ..." + foreach ($VbrScaleOutRepository in $VbrScaleOutRepositories.data) { + try { + $VbrScaleOutRepositoryName = NameCleaner $VbrScaleOutRepository.name + + $VbrSobrNotNormal = $VbrScaleOutRepository.performanceTier.performanceExtents|?{$_.status -ne "Normal"} + + if ($VbrSobrNotNormal.id) { + $VbrSobrNotNormalRepos = $VbrRepositoryTable[$VbrSobrNotNormal.id] + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.capacityGB"] + $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property capacityGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] + $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property usedSpaceGB).Sum + } + + $VbrSobrNormal = $VbrScaleOutRepository.performanceTier.performanceExtents|?{$_.status -eq "Normal"} + if ($VbrSobrNormal.id) { + $VbrSobrNormalRepos = $VbrRepositoryTable[$VbrSobrNormal.id] + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.capacityGB"] += $($VbrSobrNormalRepos|Measure-Object -Sum -Property capacityGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] += $($VbrSobrNormalRepos|Measure-Object -Sum -Property usedSpaceGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"] = $($VbrSobrNormalRepos|Measure-Object -Sum -Property freeGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.RealUsedPct"] = $($VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] * 100 / $($VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"])) + } else { + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"] = 0 + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.RealUsedPct"] = 100 + } + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Issue processing SOBR $($VbrScaleOutRepository.name)" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] No SOBR" + } try { Write-Host "$((Get-Date).ToString("o")) [INFO] VBR 5min old objectRestorePoints collect ..." From 284c5aad1738be67cbfb572f668b1ba1d9faca41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 12 May 2023 00:20:40 +0200 Subject: [PATCH 056/201] #147 #147 --- .../provisioning/dashboards/Veeam.yaml | 25 ++ .../dashboards/Veeam/Veeam_Job_Result.json | 184 ++++++++++++ .../dashboards/Veeam/Veeam_Job_Status.json | 185 ++++++++++++ .../Veeam/Veeam_Multi_Repository_Usage.json | 229 +++++++++++++++ .../dashboards/Veeam/Veeam_Restore_Point.json | 276 ++++++++++++++++++ 5 files changed, 899 insertions(+) create mode 100644 etc/grafana/provisioning/dashboards/Veeam.yaml create mode 100644 var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json create mode 100644 var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json create mode 100644 var/lib/grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json create mode 100644 var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json diff --git a/etc/grafana/provisioning/dashboards/Veeam.yaml b/etc/grafana/provisioning/dashboards/Veeam.yaml new file mode 100644 index 00000000..6e91ecd2 --- /dev/null +++ b/etc/grafana/provisioning/dashboards/Veeam.yaml @@ -0,0 +1,25 @@ +apiVersion: 1 + +providers: + # an unique provider name +- name: 'Veeam' + # org id. will default to orgId 1 if not specified + orgId: 1 + # name of the dashboard folder. Required + folder: 'Veeam Backup & Replication' + # folder UID. will be automatically generated if not specified + # CRC32 of name + folderUid: '30fb6004' + # provider type. Required + type: file + # disable dashboard deletion + disableDeletion: false + # enable dashboard editing + editable: true + # how often Grafana will scan for changed dashboards + updateIntervalSeconds: 10 + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + options: + # path to dashboard files on disk. Required + path: /var/lib/grafana/dashboards/Veeam diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json new file mode 100644 index 00000000..1d981cee --- /dev/null +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json @@ -0,0 +1,184 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "iteration": 1683841492842, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "fillOpacity": 100, + "lineWidth": 1 + }, + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "Success" + }, + "1": { + "color": "yellow", + "index": 1, + "text": "Warning" + }, + "2": { + "color": "red", + "index": 2, + "text": "Failed" + }, + "3": { + "color": "#808080", + "index": 3, + "text": "unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 31, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "hidden", + "placement": "bottom" + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "B", + "target": "aliasByNode(removeEmptySeries(veeam.vbr.$vbrserver.job.$job.result), 4)" + } + ], + "title": "Job Result", + "type": "status-history" + } + ], + "refresh": false, + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vbrserver", + "options": [], + "query": "veeam.vbr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.$vbrserver.job.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": "veeam.vbr.$vbrserver.job.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vi.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vcenter", + "options": [], + "query": "veeam.vi.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vi.$vcenter.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "cluster", + "options": [], + "query": "veeam.vi.$vcenter.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Veeam Job Result", + "uid": "oAhiTQyVk", + "version": 14, + "weekStart": "" +} \ No newline at end of file diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json new file mode 100644 index 00000000..a3094a9e --- /dev/null +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json @@ -0,0 +1,185 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "iteration": 1683841799856, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "running" + }, + "1": { + "color": "yellow", + "index": 1, + "text": "inactive" + }, + "2": { + "color": "red", + "index": 2, + "text": "disabled" + }, + "3": { + "color": "#808080", + "index": 3, + "text": "unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 31, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "mergeValues": true, + "rowHeight": 0.8, + "showValue": "never", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": "local", + "refId": "A", + "target": "aliasByNode(removeEmptySeries(veeam.vbr.$vbrserver.job.$job.status), 4)", + "textEditor": false + } + ], + "title": "Job Status", + "transformations": [], + "type": "state-timeline" + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vbrserver", + "options": [], + "query": "veeam.vbr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.$vbrserver.job.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": "veeam.vbr.$vbrserver.job.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vi.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vcenter", + "options": [], + "query": "veeam.vi.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vi.$vcenter.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "cluster", + "options": [], + "query": "veeam.vi.$vcenter.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Veeam Job Status", + "uid": "vo7u77UVk", + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json new file mode 100644 index 00000000..38b23a69 --- /dev/null +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json @@ -0,0 +1,229 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "id": null, + "iteration": 1683841959668, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": { + "capacity": "#BF1B00", + "uncommitted": "#EAB839", + "used": "#0A437C" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": false, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 4, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "repo", + "repeatDirection": "h", + "seriesOverrides": [ + { + "$$hashKey": "object:416", + "alias": "capacity", + "fill": 0, + "stack": false + }, + { + "$$hashKey": "object:708", + "alias": "RealUsedPct", + "legend": true, + "lines": false, + "stack": false, + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "A", + "target": "alias(veeam.vbr.$vbrserver.repo.$repo.usedSpaceGB, 'used')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "B", + "target": "alias(veeam.vbr.$vbrserver.repo.$repo.freeGB, 'free')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "C", + "target": "alias(veeam.vbr.$vbrserver.repo.$repo.capacityGB, 'capacity')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "D", + "target": "aliasByNode(veeam.vbr.$vbrserver.repo.$repo.RealUsedPct, 5)", + "textEditor": false + } + ], + "thresholds": [], + "timeRegions": [], + "title": "$repo usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:410", + "format": "decgbytes", + "logBase": 1, + "min": 0, + "show": true + }, + { + "$$hashKey": "object:411", + "format": "percent", + "logBase": 1, + "max": "100", + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 35, + "style": "light", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vbrserver", + "options": [], + "query": "veeam.vbr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.$vbrserver.repo.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "repo", + "options": [], + "query": "veeam.vbr.$vbrserver.repo.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "datasource": "local" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "15m", + "1h", + "2h", + "3h", + "6h", + "12h", + "24h", + "2d", + "3d", + "7d", + "15d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Veeam Multi Repository Usage", + "uid": "ofgN-MyVz", + "version": 11, + "weekStart": "" +} \ No newline at end of file diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json new file mode 100644 index 00000000..f732c3f5 --- /dev/null +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json @@ -0,0 +1,276 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "iteration": 1683841856790, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": "local", + "refId": "A", + "target": "sortByTotal(aliasByNode(veeam.vbr.$vbrserver.job.$job.objectRestorePoints, 4))" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Restore Points Per Jobs", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:37", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:38", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": "local", + "refId": "A", + "target": "groupByNode(veeam.vi.$vcenter.$cluster.vm.$vm.objectRestorePoints, 3, 'total')" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Restore Points Per Cluster", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:37", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:38", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vbrserver", + "options": [], + "query": "veeam.vbr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.$vbrserver.job.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": "veeam.vbr.$vbrserver.job.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vi.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vcenter", + "options": [], + "query": "veeam.vi.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vi.$vcenter.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "cluster", + "options": [], + "query": "veeam.vi.$vcenter.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vi.$vcenter.$cluster.vm.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vm", + "options": [], + "query": "veeam.vi.$vcenter.$cluster.vm.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query", + "datasource": "local" + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Veeam Restore Point", + "uid": "ST65OiPVk", + "version": 23, + "weekStart": "" +} \ No newline at end of file From bb3d1e9064dc4c0409ba6a3a6837a99edc0c3594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 12 May 2023 00:27:11 +0200 Subject: [PATCH 057/201] Create Veeam_Multi_SOBR_Usage.json #147 --- .../Veeam/Veeam_Multi_SOBR_Usage.json | 229 ++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json new file mode 100644 index 00000000..5c6f4026 --- /dev/null +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json @@ -0,0 +1,229 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "id": null, + "iteration": 1683841959668, + "links": [], + "liveNow": false, + "panels": [ + { + "aliasColors": { + "capacity": "#BF1B00", + "uncommitted": "#EAB839", + "used": "#0A437C" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": false, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxPerRow": 4, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "sobr", + "repeatDirection": "h", + "seriesOverrides": [ + { + "$$hashKey": "object:416", + "alias": "capacity", + "fill": 0, + "stack": false + }, + { + "$$hashKey": "object:708", + "alias": "RealUsedPct", + "legend": true, + "lines": false, + "stack": false, + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "A", + "target": "alias(veeam.vbr.$vbrserver.sobr.$sobr.usedSpaceGB, 'used')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "B", + "target": "alias(veeam.vbr.$vbrserver.sobr.$sobr.freeGB, 'free')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "C", + "target": "alias(veeam.vbr.$vbrserver.sobr.$sobr.capacityGB, 'capacity')", + "textEditor": false + }, + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "D", + "target": "aliasByNode(veeam.vbr.$vbrserver.sobr.$sobr.RealUsedPct, 5)", + "textEditor": false + } + ], + "thresholds": [], + "timeRegions": [], + "title": "$sobr usage", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:410", + "format": "decgbytes", + "logBase": 1, + "min": 0, + "show": true + }, + { + "$$hashKey": "object:411", + "format": "percent", + "logBase": 1, + "max": "100", + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "schemaVersion": 35, + "style": "light", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "vbrserver", + "options": [], + "query": "veeam.vbr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "datasource": "local" + }, + { + "allValue": "*", + "current": {}, + "definition": "veeam.vbr.$vbrserver.sobr.*", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "sobr", + "options": [], + "query": "veeam.vbr.$vbrserver.sobr.*", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query", + "datasource": "local" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "15m", + "1h", + "2h", + "3h", + "6h", + "12h", + "24h", + "2d", + "3d", + "7d", + "15d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Veeam Multi SOBR Usage", + "uid": "ofgN-NyVz", + "version": 11, + "weekStart": "" +} \ No newline at end of file From 00f6f6ab512b74089b51b0498943a9d11bb544e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 23 May 2023 00:55:57 +0200 Subject: [PATCH 058/201] Update storage-aggregation.conf --- opt/graphite/conf/storage-aggregation.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opt/graphite/conf/storage-aggregation.conf b/opt/graphite/conf/storage-aggregation.conf index ebc88992..b2510da1 100644 --- a/opt/graphite/conf/storage-aggregation.conf +++ b/opt/graphite/conf/storage-aggregation.conf @@ -56,12 +56,12 @@ aggregationMethod = last [veeamObjectRestorePoints] pattern = ^veeam.*\.objectRestorePoints xFilesFactor = 0.0 -aggregationMethod = last +aggregationMethod = sum [veeamRestorePointsCount] pattern = ^veeam.*\.restorePointsCount xFilesFactor = 0.0 -aggregationMethod = sum +aggregationMethod = last [veeamResult] pattern = ^veeam.*\.result From 15f4e1cc38bbe8d5189567a111b6e7d0c1cf9010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 24 May 2023 00:03:28 +0200 Subject: [PATCH 059/201] Create vbrcredstore.php #147 --- var/www/admin/vbrcredstore.php | 175 +++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 var/www/admin/vbrcredstore.php diff --git a/var/www/admin/vbrcredstore.php b/var/www/admin/vbrcredstore.php new file mode 100644 index 00000000..0614b490 --- /dev/null +++ b/var/www/admin/vbrcredstore.php @@ -0,0 +1,175 @@ + +

+
+

Veeam Credential Store Notes

+
    +
  • The Veeam credential store is used to store credential that will be used for Veeam Backup Server query, it uses the RESTful API for VBR (v11+)
    • +
  • Removing a VBR Server from the credential store will NOT delete any collected metrics.
    • +
  • Please refer to the project website and documentation for more information.
    • +
+
+

SexiGraf Veeam Credential Store

+ + + + + + + + + + + + + '; + if (isViEnabled($lineObjects[0])) { + echo ' '; + } else { + echo ' '; + } + if (isVsanEnabled($lineObjects[0])) { + echo ' '; + } else { + echo ' '; + } + echo ' + +'; + } + } +?> + + + + + + + + + +
VBR Server addressUsernamePasswordEnabled 
' . $lineObjects[0] . "" . $lineObjects[1] . '***********
+ + +
+ +
    '; + if (isViEnabled($lineObjects[0])) { + echo '
    • '; + } else { + echo '
    • '; + } + echo ' +
    • +
vbrpscredentials.xml +
+
 * *
+ + 0) { + $errorHappened = true; + $errorMessage = "VBR IP or FQDN is already in credential store, duplicate entry is not supported."; + } elseif (preg_match("/^([a-zA-Z0-9-_.]*)\\\\?([a-zA-Z0-9-_.]+)$|^([a-zA-Z0-9-_.]*)$|^([a-zA-Z0-9-_.]+)@([a-zA-Z0-9-_.]*)$/", $_POST["input-username"]) == 0) { + $errorHappened = true; + $errorMessage = "Wrong username format, supported format are DOMAIN\USERNAME, USERNAME, USERNAME@DOMAIN.TLD"; + } else { + exec("/usr/bin/pwsh -NonInteractive -NoProfile -f /opt/veeam/VbrConnect.ps1 -server " . escapeshellcmd($_POST["input-vbr"]) . " -username " . escapeshellcmd($_POST["input-username"]) . " -password " . escapeshellcmd($_POST["input-password"]), $null, $return_var); + if ($return_var) { + $errorHappened = true; + $errorMessage = "Wrong username/password or no answer at TCP:9419"; + } + } + + if ($errorHappened) { + echo ' '; + echo ''; + } else { + echo ' '; + echo ''; + } + break; + case "delete-vcentry": + echo '
+

+ Warning: + Confirmation needed!

+ You are about to delete entry from Veeam Credential Store for ' . $_POST["input-vbr"] . '. Are you sure about this? We mean, really sure?
+
+ + +

+
'; + echo '
'; + break; + case "delete-vcentry-confirmed": + disableVi($_POST["input-vbr"]); + disableVsan($_POST["input-vbr"]); + echo ' '; + echo ''; + break; + case "enable-vbr": + enableVi($_POST["input-vbr"]); + echo ' + '; + break; + case "disable-vbr": + disableVi($_POST["input-vbr"]); + echo ' + '; + break; + } + } +?> +
+ + + From 1862e92b6815418550bd4be5eb06318480fd47d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 24 May 2023 00:03:33 +0200 Subject: [PATCH 060/201] Update index.php #147 --- var/www/admin/index.php | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/var/www/admin/index.php b/var/www/admin/index.php index 4619f44d..2d148da9 100644 --- a/var/www/admin/index.php +++ b/var/www/admin/index.php @@ -13,7 +13,7 @@
-
Credential Store
+
vSphere Credential Store
@@ -65,7 +65,7 @@ -
+
@@ -75,12 +75,12 @@
-
Refresh Inventory
+
Refresh Inventories
- The VI Offline Inventory is automatically schedule to be updated every hours. If you want to force a refresh, you can use this section to perform update. + The VMware and Veeam inventories are automatically updated every hours. If you need to force a refresh sooner, you can use this section to force the update.
@@ -127,6 +127,28 @@ +
+
+ +
+
+
+
+ +
+
+
Veeam Credential Store
+
+
+
+
+ In this section, you'll be able to manage your Veeam Backup & Replication Credential Store. This store is used to set up your Backup Server information to allow SexiGraf query what's needed. +
+
+
+ +
+
From 9cf60501035cb5280a9b008e9d2a061f3204e0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 24 May 2023 00:03:40 +0200 Subject: [PATCH 061/201] Update credstore.php --- var/www/admin/credstore.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/www/admin/credstore.php b/var/www/admin/credstore.php index 997f08fe..32b70959 100644 --- a/var/www/admin/credstore.php +++ b/var/www/admin/credstore.php @@ -6,14 +6,14 @@ ?>

-

Credential Store Notes

+

vSphere Credential Store Notes

    -
  • The credential store is used to store credential that will be used for vCenter/ESX query, it use vSphere SDK Credential Store Library
    • +
  • The vSphere credential store is used to store credential that will be used for vCenter/ESX query, it uses vSphere SDK Credential Store Library
  • Removing a vCenter/ESX from the credential store will NOT delete any collected metrics.
  • Please refer to the project website and documentation for more information.
-

SexiGraf Credential Store

+

SexiGraf vSphere Credential Store

From 0761436bc0adc4fe2b2f5a05ea407031fbbf16aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 24 May 2023 00:57:34 +0200 Subject: [PATCH 062/201] Update helper.php #147 --- var/www/admin/helper.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/var/www/admin/helper.php b/var/www/admin/helper.php index f31863a9..9d0e7d0b 100644 --- a/var/www/admin/helper.php +++ b/var/www/admin/helper.php @@ -8,6 +8,11 @@ function isViEnabled($inputvcenter) { return (file_exists($crontabPath . "vi_" . str_replace(".", "_", $inputvcenter))); } +function isVbrEnabled($inputvbrserver) { + global $crontabPath; + return (file_exists($crontabPath . "vbr_" . str_replace(".", "_", $inputvbrserver))); +} + function isVsanEnabled($inputvcenter) { global $crontabPath; return (file_exists($crontabPath . "vsan_" . str_replace(".", "_", $inputvcenter))); @@ -22,10 +27,14 @@ function enableVi($inputvcenter) { shell_exec("sudo /bin/bash /var/www/scripts/a function enableVsan($inputvcenter) { shell_exec("sudo /bin/bash /var/www/scripts/addVsanCrontab.sh " . $inputvcenter); } +function enableVbr($inputvbrserver) { shell_exec("sudo /bin/bash /var/www/scripts/addViCrontab.sh " . $inputvbrserver); } + function disableVi($inputvcenter) { shell_exec("sudo /bin/bash /var/www/scripts/removeViCrontab.sh " . $inputvcenter); } function disableVsan($inputvcenter) { shell_exec("sudo /bin/bash /var/www/scripts/removeVsanCrontab.sh " . $inputvcenter); } +function disableVbr($inputvbrserver) { shell_exec("sudo /bin/bash /var/www/scripts/removeViCrontab.sh " . $inputvbrserver); } + function enableAutopurge($nbPurgeDays) { file_put_contents('./graphite_autopurge', $nbPurgeDays); shell_exec("sudo /bin/bash /var/www/scripts/addAutopurgeCrontab.sh " . $nbPurgeDays); From e12fdf4044fb5ed1d416d0e5ffed445c1ff98d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 24 May 2023 00:57:43 +0200 Subject: [PATCH 063/201] Update vbrcredstore.php --- var/www/admin/vbrcredstore.php | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/var/www/admin/vbrcredstore.php b/var/www/admin/vbrcredstore.php index 0614b490..34b41455 100644 --- a/var/www/admin/vbrcredstore.php +++ b/var/www/admin/vbrcredstore.php @@ -23,7 +23,7 @@ -' . $lineObjects[0] . "'; - if (isViEnabled($lineObjects[0])) { - echo ' '; - } else { - echo ' '; - } - if (isVsanEnabled($lineObjects[0])) { + if (isVbrEnabled($lineObjects[0])) { echo ' '; } else { echo ' '; @@ -52,7 +47,7 @@ Action
    '; - if (isViEnabled($lineObjects[0])) { + if (isVbrEnabled($lineObjects[0])) { echo '
    • '; } else { echo '
    • '; @@ -72,7 +67,6 @@
- @@ -138,8 +132,7 @@ echo ' '; break; case "delete-vcentry-confirmed": - disableVi($_POST["input-vbr"]); - disableVsan($_POST["input-vbr"]); + disableVbr($_POST["input-vbr"]); echo ' @@ -76,7 +76,7 @@
- ' . php_file_tree_dir_v2("/mnt/wfs/whisper", 2) . ' + ' . php_file_tree_dir_v2("/mnt/wfs/whisper", 3) . '
' . php_file_tree("/tmp", "dat") . ' @@ -109,7 +109,7 @@
- Force purge items older than days
+ Force purge deleted items older than days
From 53ae64a453b17ffb2833c9ff9f94a4dd26b3d08e Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sat, 8 Jul 2023 23:59:06 +0200 Subject: [PATCH 084/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 971dfdc5..1fecbebe 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -406,7 +406,7 @@ if ($ViServersList.count -gt 0) { } if ($VmMigratedScan -eq $true) { - Write-Host "$((Get-Date).ToString("o")) [INFO] VM Inventory differences detected, ccanning vm folders ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] VM Inventory differences detected, scanning vm folders ..." $VmFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending From 8402b064f9372b1fb09fcfc7508f2960bcd2d18f Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Tue, 11 Jul 2023 15:45:05 +0200 Subject: [PATCH 085/201] Update storage-schemas.conf --- opt/graphite/conf/storage-schemas.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opt/graphite/conf/storage-schemas.conf b/opt/graphite/conf/storage-schemas.conf index fa4721c1..487e1262 100644 --- a/opt/graphite/conf/storage-schemas.conf +++ b/opt/graphite/conf/storage-schemas.conf @@ -14,6 +14,8 @@ # Carbon's internal metrics. This entry should match what is specified in # CARBON_METRIC_PREFIX and CARBON_METRIC_INTERVAL settings +# https://m30m.github.io/whisper-calculator/ + [carbon] pattern = ^carbon\. retentions = 60:90d From 9cb7543f274ff5ecc1f680cd6290401fa2ddb26a Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sat, 15 Jul 2023 22:33:12 +0200 Subject: [PATCH 086/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 1fecbebe..63e50704 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.74" +$ScriptVersion = "0.9.75" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -406,7 +406,7 @@ if ($ViServersList.count -gt 0) { } if ($VmMigratedScan -eq $true) { - Write-Host "$((Get-Date).ToString("o")) [INFO] VM Inventory differences detected, scanning vm folders ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] VM Inventory differences detected, canning vm folders ..." $VmFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending @@ -427,7 +427,7 @@ if ($ViServersList.count -gt 0) { if ($VmFoldersDup_h) { Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated vm folders found across clusters, evaluating mobility ..." - foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 500)) { + foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 100)) { if ($VmFolders_h[$VmDup].count -lt 2) { Write-Host "$((Get-Date).ToString("o")) [EROR] VM $VmDup has less than 2 copies, skipping ..." @@ -461,11 +461,19 @@ if ($ViServersList.count -gt 0) { $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $VmDupDstVc + "/" + $VmDupDstDc + "/" + $VmDupDstClu + "/vm/" + $WspRelativePath) if (Test-Path $DstWspFullPath) { try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($VmDupWsp2Mv.FullName) to $DstWspFullPath" - $VmDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $VmDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($VmDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $DstWspFullPath whisper-info" + $VmDupWspSrcInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $DstWspFullPath" + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $($VmDupWsp2Mv.FullName) whisper-info" + $VmDupWsp2MvInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $($VmDupWsp2Mv.FullName)" + + if (Compare-Object $VmDupWspSrcInfo $VmDupWsp2MvInfo) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Resizing $($VmDupWsp2Mv.FullName) and $DstWspFullPath" + $VmDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $VmDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($VmDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($VmDupWsp2Mv.FullName) to $DstWspFullPath" + } + $VmDupWsp2MvMerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($VmDupWsp2Mv.FullName) $DstWspFullPath" - } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." continue @@ -500,7 +508,7 @@ if ($ViServersList.count -gt 0) { $EsxFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xEsxotioned Esxs aka DstEsxMigratedEvent ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xEsxmotioned Esxs aka DstEsxMigratedEvent ..." $EsxFolders_h = @{} $EsxFoldersDup_h = @{} From 5f5a6c114fb9fbfe0850156bc59b22bc2119840e Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sat, 15 Jul 2023 23:30:33 +0200 Subject: [PATCH 087/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 223 ++++++++++++++++------------ 1 file changed, 130 insertions(+), 93 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 63e50704..87128f6d 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.75" +$ScriptVersion = "0.9.76" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -363,15 +363,21 @@ if ($ViServersList.count -gt 0) { if ($ViVmsInfos) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] Building Vm Inventory CSV ..." - $ViVmsInfosCsv = $ViVmsInfos|Export-Csv -NoTypeInformation -ErrorAction Stop + $ViVmsInfosCsv = $ViVmsInfos|ConvertTo-Csv -NoTypeInformation -ErrorAction Stop Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for vCenter,Cluster,VM differences with previous inventory file ..." - if (Test-Path /mnt/wfs/inventory/ViVmInventory.csv) { - if ($ViVmsInfosCsvDiff = Compare-Object $(Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop) $ViVmsInfosCsv -Property vCenter,Cluster,VM) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Differences detected with previous inventory file ..." - $VmMigratedScan = $true + if (Test-Path -Path /mnt/wfs/inventory/ViVmInventory.csv) { + $ViVmsInfosCsvBak = Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop + if ($ViVmsInfosCsvBak) { + if (Compare-Object $ViVmsInfosCsvBak $ViVmsInfos -Property vCenter,Cluster,VM) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Differences detected with previous inventory file ..." + $VmMigratedScan = $true + } else { + $VmMigratedScan = $false + Write-Host "$((Get-Date).ToString("o")) [INFO] No differences detected with previous inventory file ..." + } } else { - $VmMigratedScan = $false - Write-Host "$((Get-Date).ToString("o")) [INFO] No differences detected with previous inventory file ..." + Write-Host "$((Get-Date).ToString("o")) [EROR] Empty Vm Inventory file" + $VmMigratedScan = $true } } else { Write-Host "$((Get-Date).ToString("o")) [EROR] No existing Vm Inventory file" @@ -380,17 +386,38 @@ if ($ViServersList.count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory file ..." $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.csv -Force -ErrorAction Stop } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] VM Inventory issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } } if ($ViEsxsInfos) { try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Esx Inventory files ..." - $ViEsxsInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force -ErrorAction Stop + Write-Host "$((Get-Date).ToString("o")) [INFO] Building ESX Inventory CSV ..." + $ViEsxsInfosCsv = $ViEsxsInfos|ConvertTo-Csv -NoTypeInformation -ErrorAction Stop + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for vCenter,Cluster,ESX differences with previous inventory file ..." + if (Test-Path -Path /mnt/wfs/inventory/ViEsxInventory.csv) { + $ViEsxsInfosCsvBak = Import-Csv -Path /mnt/wfs/inventory/ViEsxInventory.csv -ErrorAction Stop + if ($ViEsxsInfosCsvBak) { + if (Compare-Object $ViEsxsInfosCsvBak $ViEsxsInfos -Property vCenter,Cluster,ESX) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Differences detected with previous inventory file ..." + $EsxMigratedScan = $true + } else { + $EsxMigratedScan = $false + Write-Host "$((Get-Date).ToString("o")) [INFO] No differences detected with previous inventory file ..." + } + } else { + Write-Host "$((Get-Date).ToString("o")) [EROR] Empty ESX Inventory file" + $EsxMigratedScan = $true + } + } else { + Write-Host "$((Get-Date).ToString("o")) [EROR] No existing ESX Inventory file" + $EsxMigratedScan = $true + } + Write-Host "$((Get-Date).ToString("o")) [INFO] Writing ESX Inventory file ..." + $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force -ErrorAction Stop } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX Export-Csv issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] ESX Inventory issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } } @@ -406,7 +433,7 @@ if ($ViServersList.count -gt 0) { } if ($VmMigratedScan -eq $true) { - Write-Host "$((Get-Date).ToString("o")) [INFO] VM Inventory differences detected, canning vm folders ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] VM Inventory differences detected, scanning vm folders ..." $VmFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending @@ -426,8 +453,8 @@ if ($ViServersList.count -gt 0) { } if ($VmFoldersDup_h) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated vm folders found across clusters, evaluating mobility ..." - foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 100)) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicate vm folders found across clusters, evaluating mobility ..." + foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 500)) { if ($VmFolders_h[$VmDup].count -lt 2) { Write-Host "$((Get-Date).ToString("o")) [EROR] VM $VmDup has less than 2 copies, skipping ..." @@ -504,93 +531,103 @@ if ($ViServersList.count -gt 0) { } } - Write-Host "$((Get-Date).ToString("o")) [INFO] Scanning Esx folders ..." + if ($EsxMigratedScan) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Inventory differences detected, scanning Esx folders ..." - $EsxFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending - - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xEsxmotioned Esxs aka DstEsxMigratedEvent ..." - - $EsxFolders_h = @{} - $EsxFoldersDup_h = @{} - foreach ($EsxFolder in $EsxFolders) { - if (!$EsxFolders_h[$EsxFolder.basename]) { - $EsxFolders_h.add($EsxFolder.basename,@($EsxFolder)) - } else { - $EsxFolders_h[$EsxFolder.basename] += $EsxFolder - if (!$EsxFoldersDup_h[$EsxFolder.basename]) { - $EsxFoldersDup_h.add($EsxFolder.basename,"1") + $EsxFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending + + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for *xvmotioned* ESXs aka DstEsxMigratedEvent ..." + + $EsxFolders_h = @{} + $EsxFoldersDup_h = @{} + foreach ($EsxFolder in $EsxFolders) { + if (!$EsxFolders_h[$EsxFolder.basename]) { + $EsxFolders_h.add($EsxFolder.basename,@($EsxFolder)) + } else { + $EsxFolders_h[$EsxFolder.basename] += $EsxFolder + if (!$EsxFoldersDup_h[$EsxFolder.basename]) { + $EsxFoldersDup_h.add($EsxFolder.basename,"1") + } } } - } - - if ($EsxFoldersDup_h) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicated Esx folders found across clusters, evaluating mobility ..." - foreach ($EsxDup in $($EsxFoldersDup_h.keys|Select-Object -first 50)) { - - if ($EsxFolders_h[$EsxDup].count -lt 2) { - Write-Host "$((Get-Date).ToString("o")) [EROR] Esx $EsxDup has less than 2 copies, skipping ..." - continue - } elseif ($EsxFolders_h[$EsxDup].count -gt 2) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has more than 2 copies ..." - } - - $EsxDupFolders = $EsxFolders_h[$EsxDup] - $EsxDupSrcDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[1] - $EsxDupDstDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[0] - - try { - $EsxDupSrcWsp = Get-Item $($EsxDupSrcDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - $EsxDupDstWsp = Get-Item $($EsxDupDstDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for Esx $EsxDup ..." - continue - } - - if (($EsxDupDstDir.CreationTimeUtc -gt $EsxDupSrcDir.CreationTimeUtc) -and ($EsxDupDstWsp.LastWriteTimeUtc - $EsxDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { - - $EsxDupDstVc = $EsxDupDstDir.FullName.split("/")[5] - $EsxDupDstDc = $EsxDupDstDir.FullName.split("/")[6] - $EsxDupDstClu = $EsxDupDstDir.FullName.split("/")[7] - $EsxDupSrcClu = $EsxDupSrcDir.FullName.split("/")[7] - Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has been moved from cluster $EsxDupSrcClu to cluster $EsxDupDstClu a while ago, merging metrics to the new destination if possible ..." - $EsxDupWsps2Mv = Get-ChildItem -Recurse $EsxDupSrcDir.FullName -Filter *.wsp - foreach ($EsxDupWsp2Mv in $EsxDupWsps2Mv) { - $WspRelativePath = $($EsxDupWsp2Mv.FullName -split "/Esx/")[1] - $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $EsxDupDstVc + "/" + $EsxDupDstDc + "/" + $EsxDupDstClu + "/esx/" + $WspRelativePath) - if (Test-Path $DstWspFullPath) { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($EsxDupWsp2Mv.FullName) to $DstWspFullPath" - $EsxDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $EsxDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($EsxDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $EsxDupWsp2MEsxerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($EsxDupWsp2Mv.FullName) $DstWspFullPath" - - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." - continue - } - } else { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($EsxDupWsp2Mv.FullName)" - $EsxDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop - $EsxDupWspMv = Move-Item $EsxDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." - continue - } - } + + if ($EsxFoldersDup_h) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicate Esx folders found across clusters, evaluating mobility ..." + foreach ($EsxDup in $($EsxFoldersDup_h.keys|Select-Object -first 50)) { + + if ($EsxFolders_h[$EsxDup].count -lt 2) { + Write-Host "$((Get-Date).ToString("o")) [EROR] Esx $EsxDup has less than 2 copies, skipping ..." + continue + } elseif ($EsxFolders_h[$EsxDup].count -gt 2) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has more than 2 copies ..." } - Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($EsxDupSrcDir.FullName)" + + $EsxDupFolders = $EsxFolders_h[$EsxDup] + $EsxDupSrcDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[1] + $EsxDupDstDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[0] + try { - Remove-Item -Recurse $($EsxDupSrcDir.FullName) -Force -ErrorAction Stop + $EsxDupSrcWsp = Get-Item $($EsxDupSrcDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $EsxDupDstWsp = Get-Item $($EsxDupDstDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($EsxDupSrcDir.FullName) issue ..." + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for Esx $EsxDup ..." + continue + } + + if (($EsxDupDstDir.CreationTimeUtc -gt $EsxDupSrcDir.CreationTimeUtc) -and ($EsxDupDstWsp.LastWriteTimeUtc - $EsxDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { + + $EsxDupDstVc = $EsxDupDstDir.FullName.split("/")[5] + $EsxDupDstDc = $EsxDupDstDir.FullName.split("/")[6] + $EsxDupDstClu = $EsxDupDstDir.FullName.split("/")[7] + $EsxDupSrcClu = $EsxDupSrcDir.FullName.split("/")[7] + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has been moved from cluster $EsxDupSrcClu to cluster $EsxDupDstClu a while ago, merging metrics to the new destination if possible ..." + $EsxDupWsps2Mv = Get-ChildItem -Recurse $EsxDupSrcDir.FullName -Filter *.wsp + foreach ($EsxDupWsp2Mv in $EsxDupWsps2Mv) { + $WspRelativePath = $($EsxDupWsp2Mv.FullName -split "/Esx/")[1] + $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $EsxDupDstVc + "/" + $EsxDupDstDc + "/" + $EsxDupDstClu + "/esx/" + $WspRelativePath) + if (Test-Path $DstWspFullPath) { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $DstWspFullPath whisper-info" + $VmDupWspSrcInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $DstWspFullPath" + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $($EsxDupWsp2Mv.FullName) whisper-info" + $EsxDupWsp2MvInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $($EsxDupWsp2Mv.FullName)" + + if (Compare-Object $VmDupWspSrcInfo $EsxDupWsp2MvInfo) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Resizing $($EsxDupWsp2Mv.FullName) and $DstWspFullPath" + $EsxDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $EsxDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($EsxDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + } + + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($EsxDupWsp2Mv.FullName) to $DstWspFullPath" + $EsxDupWsp2MEsxerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($EsxDupWsp2Mv.FullName) $DstWspFullPath" + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." + continue + } + } else { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($EsxDupWsp2Mv.FullName)" + $EsxDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop + $EsxDupWspMv = Move-Item $EsxDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." + continue + } + } + } + Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($EsxDupSrcDir.FullName)" + try { + Remove-Item -Recurse $($EsxDupSrcDir.FullName) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($EsxDupSrcDir.FullName) issue ..." + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup move is too recent, has clones or has come back to its original location ..." } - } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup move is too recent, has clones or has come back to its original location ..." } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated Esx folders found" } - } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated Esx folders found" } Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf has left the building ..." From c050c0f72bc200fc5052a71aa09f515a33de6988 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Wed, 19 Jul 2023 19:16:15 +0200 Subject: [PATCH 088/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 87128f6d..61ad601a 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.76" +$ScriptVersion = "0.9.77" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -432,6 +432,8 @@ if ($ViServersList.count -gt 0) { } } + #TODO force scan switch + if ($VmMigratedScan -eq $true) { Write-Host "$((Get-Date).ToString("o")) [INFO] VM Inventory differences detected, scanning vm folders ..." @@ -452,9 +454,9 @@ if ($ViServersList.count -gt 0) { } } - if ($VmFoldersDup_h) { + if ($VmFoldersDup_h.Count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicate vm folders found across clusters, evaluating mobility ..." - foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 500)) { + foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 100)) { if ($VmFolders_h[$VmDup].count -lt 2) { Write-Host "$((Get-Date).ToString("o")) [EROR] VM $VmDup has less than 2 copies, skipping ..." @@ -497,9 +499,9 @@ if ($ViServersList.count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] Resizing $($VmDupWsp2Mv.FullName) and $DstWspFullPath" $VmDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" $VmDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($VmDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($VmDupWsp2Mv.FullName) to $DstWspFullPath" } + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($VmDupWsp2Mv.FullName) to $DstWspFullPath" $VmDupWsp2MvMerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($VmDupWsp2Mv.FullName) $DstWspFullPath" } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." @@ -551,9 +553,9 @@ if ($ViServersList.count -gt 0) { } } - if ($EsxFoldersDup_h) { + if ($EsxFoldersDup_h.Count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicate Esx folders found across clusters, evaluating mobility ..." - foreach ($EsxDup in $($EsxFoldersDup_h.keys|Select-Object -first 50)) { + foreach ($EsxDup in $($EsxFoldersDup_h.keys|Select-Object -first 10)) { if ($EsxFolders_h[$EsxDup].count -lt 2) { Write-Host "$((Get-Date).ToString("o")) [EROR] Esx $EsxDup has less than 2 copies, skipping ..." From 2ae9064e2965206982a310d5ae1efc43924a152f Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 20 Jul 2023 22:46:31 +0200 Subject: [PATCH 089/201] Create xMotionGarbageCollector.ps1 #325 --- opt/sexigraf/xMotionGarbageCollector.ps1 | 254 +++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 opt/sexigraf/xMotionGarbageCollector.ps1 diff --git a/opt/sexigraf/xMotionGarbageCollector.ps1 b/opt/sexigraf/xMotionGarbageCollector.ps1 new file mode 100644 index 00000000..a02eac83 --- /dev/null +++ b/opt/sexigraf/xMotionGarbageCollector.ps1 @@ -0,0 +1,254 @@ +#!/usr/bin/pwsh -Command +# + +$ScriptVersion = "0.9.1" + +$ErrorActionPreference = "SilentlyContinue" +$WarningPreference = "SilentlyContinue" +(Get-Process -Id $pid).PriorityClass = 'BelowNormal' + +function AltAndCatchFire { + Param($ExitReason) + Write-Host "$((Get-Date).ToString("o")) [EROR] $ExitReason" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + Stop-Transcript + exit +} + +try { + Start-Transcript -Path "/var/log/sexigraf/xMotionGarbageCollector.log" -Append -Confirm:$false -Force -UseMinimalHeader + Write-Host "$((Get-Date).ToString("o")) [INFO] xMotionGarbageCollector v$ScriptVersion" +} catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] xMotionGarbageCollector logging failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + exit +} + +try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for another xMotionGarbageCollector ..." + $DupViVmInventoryProcess = Get-PSHostProcessInfo|%{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}|?{$_ -match "xMotionGarbageCollector"} + # https://github.com/PowerShell/PowerShell/issues/13944 + if (($DupViVmInventoryProcess|Measure-Object).Count -gt 1) { + $DupViVmInventoryProcessId = (Get-PSHostProcessInfo|?{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '|?{$_ -match "xMotionGarbageCollector"}}).ProcessId[0] + $DupViVmInventoryProcessTime = [INT32](ps -p $DupViVmInventoryProcessId -o etimes).split()[-1] + if ($DupViVmInventoryProcessTime -gt 21600) { + Write-Host "$((Get-Date).ToString("o")) [WARN] xMotionGarbageCollector is already running for more than 6 hours!" + Write-Host "$((Get-Date).ToString("o")) [WARN] Killing stunned xMotionGarbageCollector" + Stop-Process -Id $DupViVmInventoryProcessId -Force + } else { + AltAndCatchFire "xMotionGarbageCollector is already running!" + } + } +} catch { + AltAndCatchFire "xMotionGarbageCollector process lookup failure" +} + +Write-Host "$((Get-Date).ToString("o")) [INFO] Collecting VM folders ..." + +try { + $VmFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending +} catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Collecting VM folders issue ..." +} + +if ($VmFolders) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xvmotioned vms aka DstVmMigratedEvent ..." + + $VmFolders_h = @{} + $VmFoldersDup_h = @{} + foreach ($VmFolder in $VmFolders) { + if (!$VmFolders_h[$VmFolder.basename]) { + $VmFolders_h.add($VmFolder.basename,@($VmFolder)) + } else { + $VmFolders_h[$VmFolder.basename] += $VmFolder + if (!$VmFoldersDup_h[$VmFolder.basename]) { + $VmFoldersDup_h.add($VmFolder.basename,"1") + } + } + } + + if ($VmFoldersDup_h.Count -gt 0) { + Write-Host "$((Get-Date).ToString("o")) [INFO] $($VmFoldersDup_h.Count) Duplicate vm folders found across clusters, evaluating mobility ..." + foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 100)) { + + if ($VmFolders_h[$VmDup].count -lt 2) { + Write-Host "$((Get-Date).ToString("o")) [EROR] VM $VmDup has less than 2 copies, skipping ..." + continue + } elseif ($VmFolders_h[$VmDup].count -gt 2) { + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has more than 2 copies ..." + } + + $VmDupFolders = $VmFolders_h[$VmDup] + $VmDupSrcDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[1] + $VmDupDstDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[0] + + try { + $VmDupSrcWsp = Get-Item $($VmDupSrcDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $VmDupDstWsp = Get-Item $($VmDupDstDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for vm $VmDup ..." + continue + } + + if (($VmDupDstDir.CreationTimeUtc -gt $VmDupSrcDir.CreationTimeUtc) -and ($VmDupDstWsp.LastWriteTimeUtc - $VmDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { + + $VmDupDstVc = $VmDupDstDir.FullName.split("/")[5] + $VmDupDstDc = $VmDupDstDir.FullName.split("/")[6] + $VmDupDstClu = $VmDupDstDir.FullName.split("/")[7] + $VmDupSrcClu = $VmDupSrcDir.FullName.split("/")[7] + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has been moved from cluster $VmDupSrcClu to cluster $VmDupDstClu a while ago, merging metrics to the new destination if possible ..." + $VmDupWsps2Mv = Get-ChildItem -Recurse $VmDupSrcDir.FullName -Filter *.wsp + foreach ($VmDupWsp2Mv in $VmDupWsps2Mv) { + $WspRelativePath = $($VmDupWsp2Mv.FullName -split "/vm/")[1] + $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $VmDupDstVc + "/" + $VmDupDstDc + "/" + $VmDupDstClu + "/vm/" + $WspRelativePath) + if (Test-Path $DstWspFullPath) { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $DstWspFullPath whisper-info" + $VmDupWspSrcInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $DstWspFullPath" + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $($VmDupWsp2Mv.FullName) whisper-info" + $VmDupWsp2MvInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $($VmDupWsp2Mv.FullName)" + + if (Compare-Object $VmDupWspSrcInfo $VmDupWsp2MvInfo) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Resizing $($VmDupWsp2Mv.FullName) and $DstWspFullPath" + $VmDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $VmDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($VmDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + } + + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($VmDupWsp2Mv.FullName) to $DstWspFullPath" + $VmDupWsp2MvMerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($VmDupWsp2Mv.FullName) $DstWspFullPath" + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." + continue + } + } else { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($VmDupWsp2Mv.FullName)" + $VmDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop + $VmDupWspMv = Move-Item $VmDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." + continue + } + } + } + Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($VmDupSrcDir.FullName)" + try { + Remove-Item -Recurse $($VmDupSrcDir.FullName) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($VmDupSrcDir.FullName) issue ..." + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup move is too recent, has clones or has come back to its original location ..." + # TODO deal with infinite clones + } + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated vm folders found" + } +} + +Write-Host "$((Get-Date).ToString("o")) [INFO] Collecting ESX folders ..." + +try { + $EsxFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending +} catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Collecting ESX folders issue ..." +} + +if ($EsxFolders) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for *xvmotioned* ESXs aka DstEsxMigratedEvent ..." + + $EsxFolders_h = @{} + $EsxFoldersDup_h = @{} + foreach ($EsxFolder in $EsxFolders) { + if (!$EsxFolders_h[$EsxFolder.basename]) { + $EsxFolders_h.add($EsxFolder.basename,@($EsxFolder)) + } else { + $EsxFolders_h[$EsxFolder.basename] += $EsxFolder + if (!$EsxFoldersDup_h[$EsxFolder.basename]) { + $EsxFoldersDup_h.add($EsxFolder.basename,"1") + } + } + } + + if ($EsxFoldersDup_h.Count -gt 0) { + Write-Host "$((Get-Date).ToString("o")) [INFO] $($EsxFoldersDup_h.Count) Duplicate Esx folders found across clusters, evaluating mobility ..." + foreach ($EsxDup in $($EsxFoldersDup_h.keys|Select-Object -first 10)) { + + if ($EsxFolders_h[$EsxDup].count -lt 2) { + Write-Host "$((Get-Date).ToString("o")) [EROR] Esx $EsxDup has less than 2 copies, skipping ..." + continue + } elseif ($EsxFolders_h[$EsxDup].count -gt 2) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has more than 2 copies ..." + } + + $EsxDupFolders = $EsxFolders_h[$EsxDup] + $EsxDupSrcDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[1] + $EsxDupDstDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[0] + + try { + $EsxDupSrcWsp = Get-Item $($EsxDupSrcDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + $EsxDupDstWsp = Get-Item $($EsxDupDstDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for Esx $EsxDup ..." + continue + } + + if (($EsxDupDstDir.CreationTimeUtc -gt $EsxDupSrcDir.CreationTimeUtc) -and ($EsxDupDstWsp.LastWriteTimeUtc - $EsxDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { + + $EsxDupDstVc = $EsxDupDstDir.FullName.split("/")[5] + $EsxDupDstDc = $EsxDupDstDir.FullName.split("/")[6] + $EsxDupDstClu = $EsxDupDstDir.FullName.split("/")[7] + $EsxDupSrcClu = $EsxDupSrcDir.FullName.split("/")[7] + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has been moved from cluster $EsxDupSrcClu to cluster $EsxDupDstClu a while ago, merging metrics to the new destination if possible ..." + $EsxDupWsps2Mv = Get-ChildItem -Recurse $EsxDupSrcDir.FullName -Filter *.wsp + foreach ($EsxDupWsp2Mv in $EsxDupWsps2Mv) { + $WspRelativePath = $($EsxDupWsp2Mv.FullName -split "/Esx/")[1] + $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $EsxDupDstVc + "/" + $EsxDupDstDc + "/" + $EsxDupDstClu + "/esx/" + $WspRelativePath) + if (Test-Path $DstWspFullPath) { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $DstWspFullPath whisper-info" + $VmDupWspSrcInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $DstWspFullPath" + Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $($EsxDupWsp2Mv.FullName) whisper-info" + $EsxDupWsp2MvInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $($EsxDupWsp2Mv.FullName)" + + if (Compare-Object $VmDupWspSrcInfo $EsxDupWsp2MvInfo) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Resizing $($EsxDupWsp2Mv.FullName) and $DstWspFullPath" + $EsxDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $EsxDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($EsxDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + } + + Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($EsxDupWsp2Mv.FullName) to $DstWspFullPath" + $EsxDupWsp2MEsxerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($EsxDupWsp2Mv.FullName) $DstWspFullPath" + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." + continue + } + } else { + try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($EsxDupWsp2Mv.FullName)" + $EsxDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop + $EsxDupWspMv = Move-Item $EsxDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." + continue + } + } + } + Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($EsxDupSrcDir.FullName)" + try { + Remove-Item -Recurse $($EsxDupSrcDir.FullName) -Force -ErrorAction Stop + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($EsxDupSrcDir.FullName) issue ..." + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup move is too recent, has clones or has come back to its original location ..." + } + } + } else { + Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated Esx folders found" + } +} + +Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf xMotionGarbageCollector has left the building ..." \ No newline at end of file From 2e7c599529a0c12aece9208f0928d198aef1c316 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 20 Jul 2023 22:46:40 +0200 Subject: [PATCH 090/201] Create xMotionGarbageCollector #325 --- etc/cron.d/xMotionGarbageCollector | 1 + 1 file changed, 1 insertion(+) create mode 100644 etc/cron.d/xMotionGarbageCollector diff --git a/etc/cron.d/xMotionGarbageCollector b/etc/cron.d/xMotionGarbageCollector new file mode 100644 index 00000000..99f91d18 --- /dev/null +++ b/etc/cron.d/xMotionGarbageCollector @@ -0,0 +1 @@ +37 1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/xMotionGarbageCollector.ps1 >/dev/null 2>&1 From 54083f95f8c76294d5356918588da47dcf52068a Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 20 Jul 2023 22:46:53 +0200 Subject: [PATCH 091/201] Update storage-schemas.conf --- opt/graphite/conf/storage-schemas.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opt/graphite/conf/storage-schemas.conf b/opt/graphite/conf/storage-schemas.conf index 487e1262..d04ff74e 100644 --- a/opt/graphite/conf/storage-schemas.conf +++ b/opt/graphite/conf/storage-schemas.conf @@ -58,4 +58,4 @@ retentions = 5m:7d,10m:30d,15m:90d,30m:120d,120m:1y [anything_else] pattern = .* -retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y +retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y \ No newline at end of file From 782224d1b435f1991640befbcb2b54c333974f12 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 20 Jul 2023 22:47:34 +0200 Subject: [PATCH 092/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 280 ++++------------------------ 1 file changed, 40 insertions(+), 240 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 61ad601a..d1726879 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.77" +$ScriptVersion = "0.9.80" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -364,25 +364,25 @@ if ($ViServersList.count -gt 0) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] Building Vm Inventory CSV ..." $ViVmsInfosCsv = $ViVmsInfos|ConvertTo-Csv -NoTypeInformation -ErrorAction Stop - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for vCenter,Cluster,VM differences with previous inventory file ..." - if (Test-Path -Path /mnt/wfs/inventory/ViVmInventory.csv) { - $ViVmsInfosCsvBak = Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop - if ($ViVmsInfosCsvBak) { - if (Compare-Object $ViVmsInfosCsvBak $ViVmsInfos -Property vCenter,Cluster,VM) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Differences detected with previous inventory file ..." - $VmMigratedScan = $true - } else { - $VmMigratedScan = $false - Write-Host "$((Get-Date).ToString("o")) [INFO] No differences detected with previous inventory file ..." - } - } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] Empty Vm Inventory file" - $VmMigratedScan = $true - } - } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] No existing Vm Inventory file" - $VmMigratedScan = $true - } + # Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for vCenter,Cluster,VM differences with previous inventory file ..." + # if (Test-Path -Path /mnt/wfs/inventory/ViVmInventory.csv) { + # $ViVmsInfosCsvBak = Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop + # if ($ViVmsInfosCsvBak) { + # if (Compare-Object $ViVmsInfosCsvBak $ViVmsInfos -Property vCenter,Cluster,VM) { + # Write-Host "$((Get-Date).ToString("o")) [INFO] Differences detected with previous inventory file ..." + # $VmMigratedScan = $true + # } else { + # $VmMigratedScan = $false + # Write-Host "$((Get-Date).ToString("o")) [INFO] No differences detected with previous inventory file ..." + # } + # } else { + # Write-Host "$((Get-Date).ToString("o")) [EROR] Empty Vm Inventory file" + # $VmMigratedScan = $true + # } + # } else { + # Write-Host "$((Get-Date).ToString("o")) [EROR] No existing Vm Inventory file" + # $VmMigratedScan = $true + # } Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory file ..." $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.csv -Force -ErrorAction Stop } catch { @@ -395,25 +395,25 @@ if ($ViServersList.count -gt 0) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] Building ESX Inventory CSV ..." $ViEsxsInfosCsv = $ViEsxsInfos|ConvertTo-Csv -NoTypeInformation -ErrorAction Stop - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for vCenter,Cluster,ESX differences with previous inventory file ..." - if (Test-Path -Path /mnt/wfs/inventory/ViEsxInventory.csv) { - $ViEsxsInfosCsvBak = Import-Csv -Path /mnt/wfs/inventory/ViEsxInventory.csv -ErrorAction Stop - if ($ViEsxsInfosCsvBak) { - if (Compare-Object $ViEsxsInfosCsvBak $ViEsxsInfos -Property vCenter,Cluster,ESX) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Differences detected with previous inventory file ..." - $EsxMigratedScan = $true - } else { - $EsxMigratedScan = $false - Write-Host "$((Get-Date).ToString("o")) [INFO] No differences detected with previous inventory file ..." - } - } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] Empty ESX Inventory file" - $EsxMigratedScan = $true - } - } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] No existing ESX Inventory file" - $EsxMigratedScan = $true - } + # Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for vCenter,Cluster,ESX differences with previous inventory file ..." + # if (Test-Path -Path /mnt/wfs/inventory/ViEsxInventory.csv) { + # $ViEsxsInfosCsvBak = Import-Csv -Path /mnt/wfs/inventory/ViEsxInventory.csv -ErrorAction Stop + # if ($ViEsxsInfosCsvBak) { + # if (Compare-Object $ViEsxsInfosCsvBak $ViEsxsInfos -Property vCenter,Cluster,ESX) { + # Write-Host "$((Get-Date).ToString("o")) [INFO] Differences detected with previous inventory file ..." + # $EsxMigratedScan = $true + # } else { + # $EsxMigratedScan = $false + # Write-Host "$((Get-Date).ToString("o")) [INFO] No differences detected with previous inventory file ..." + # } + # } else { + # Write-Host "$((Get-Date).ToString("o")) [EROR] Empty ESX Inventory file" + # $EsxMigratedScan = $true + # } + # } else { + # Write-Host "$((Get-Date).ToString("o")) [EROR] No existing ESX Inventory file" + # $EsxMigratedScan = $true + # } Write-Host "$((Get-Date).ToString("o")) [INFO] Writing ESX Inventory file ..." $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force -ErrorAction Stop } catch { @@ -432,207 +432,7 @@ if ($ViServersList.count -gt 0) { } } - #TODO force scan switch - - if ($VmMigratedScan -eq $true) { - Write-Host "$((Get-Date).ToString("o")) [INFO] VM Inventory differences detected, scanning vm folders ..." - - $VmFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/vm/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending - - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for xvmotioned vms aka DstVmMigratedEvent ..." - - $VmFolders_h = @{} - $VmFoldersDup_h = @{} - foreach ($VmFolder in $VmFolders) { - if (!$VmFolders_h[$VmFolder.basename]) { - $VmFolders_h.add($VmFolder.basename,@($VmFolder)) - } else { - $VmFolders_h[$VmFolder.basename] += $VmFolder - if (!$VmFoldersDup_h[$VmFolder.basename]) { - $VmFoldersDup_h.add($VmFolder.basename,"1") - } - } - } - - if ($VmFoldersDup_h.Count -gt 0) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicate vm folders found across clusters, evaluating mobility ..." - foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 100)) { - - if ($VmFolders_h[$VmDup].count -lt 2) { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $VmDup has less than 2 copies, skipping ..." - continue - } elseif ($VmFolders_h[$VmDup].count -gt 2) { - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has more than 2 copies ..." - } - - $VmDupFolders = $VmFolders_h[$VmDup] - $VmDupSrcDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[1] - $VmDupDstDir = $($VmDupFolders|Sort-Object CreationTimeUtc -Descending)[0] - - try { - $VmDupSrcWsp = Get-Item $($VmDupSrcDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - $VmDupDstWsp = Get-Item $($VmDupDstDir.FullName + "/storage/committed.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for vm $VmDup ..." - continue - } - - if (($VmDupDstDir.CreationTimeUtc -gt $VmDupSrcDir.CreationTimeUtc) -and ($VmDupDstWsp.LastWriteTimeUtc - $VmDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { - - $VmDupDstVc = $VmDupDstDir.FullName.split("/")[5] - $VmDupDstDc = $VmDupDstDir.FullName.split("/")[6] - $VmDupDstClu = $VmDupDstDir.FullName.split("/")[7] - $VmDupSrcClu = $VmDupSrcDir.FullName.split("/")[7] - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup has been moved from cluster $VmDupSrcClu to cluster $VmDupDstClu a while ago, merging metrics to the new destination if possible ..." - $VmDupWsps2Mv = Get-ChildItem -Recurse $VmDupSrcDir.FullName -Filter *.wsp - foreach ($VmDupWsp2Mv in $VmDupWsps2Mv) { - $WspRelativePath = $($VmDupWsp2Mv.FullName -split "/vm/")[1] - $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $VmDupDstVc + "/" + $VmDupDstDc + "/" + $VmDupDstClu + "/vm/" + $WspRelativePath) - if (Test-Path $DstWspFullPath) { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $DstWspFullPath whisper-info" - $VmDupWspSrcInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $DstWspFullPath" - Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $($VmDupWsp2Mv.FullName) whisper-info" - $VmDupWsp2MvInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $($VmDupWsp2Mv.FullName)" - - if (Compare-Object $VmDupWspSrcInfo $VmDupWsp2MvInfo) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Resizing $($VmDupWsp2Mv.FullName) and $DstWspFullPath" - $VmDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $VmDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($VmDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - } - - Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($VmDupWsp2Mv.FullName) to $DstWspFullPath" - $VmDupWsp2MvMerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($VmDupWsp2Mv.FullName) $DstWspFullPath" - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." - continue - } - } else { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($VmDupWsp2Mv.FullName)" - $VmDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop - $VmDupWspMv = Move-Item $VmDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($VmDupWsp2Mv.FullName) moving issue ..." - continue - } - } - } - Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($VmDupSrcDir.FullName)" - try { - Remove-Item -Recurse $($VmDupSrcDir.FullName) -Force -ErrorAction Stop - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($VmDupSrcDir.FullName) issue ..." - } - } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup move is too recent, has clones or has come back to its original location ..." - } - } - } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated vm folders found" - } - } - - if ($EsxMigratedScan) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Inventory differences detected, scanning Esx folders ..." - - $EsxFolders = Get-ChildItem -Directory /mnt/wfs/whisper/vmw/*/*/*/esx/*|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc|Sort-Object LastAccessTimeUtc -Descending - - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for *xvmotioned* ESXs aka DstEsxMigratedEvent ..." - - $EsxFolders_h = @{} - $EsxFoldersDup_h = @{} - foreach ($EsxFolder in $EsxFolders) { - if (!$EsxFolders_h[$EsxFolder.basename]) { - $EsxFolders_h.add($EsxFolder.basename,@($EsxFolder)) - } else { - $EsxFolders_h[$EsxFolder.basename] += $EsxFolder - if (!$EsxFoldersDup_h[$EsxFolder.basename]) { - $EsxFoldersDup_h.add($EsxFolder.basename,"1") - } - } - } - - if ($EsxFoldersDup_h.Count -gt 0) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Duplicate Esx folders found across clusters, evaluating mobility ..." - foreach ($EsxDup in $($EsxFoldersDup_h.keys|Select-Object -first 10)) { - - if ($EsxFolders_h[$EsxDup].count -lt 2) { - Write-Host "$((Get-Date).ToString("o")) [EROR] Esx $EsxDup has less than 2 copies, skipping ..." - continue - } elseif ($EsxFolders_h[$EsxDup].count -gt 2) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has more than 2 copies ..." - } - - $EsxDupFolders = $EsxFolders_h[$EsxDup] - $EsxDupSrcDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[1] - $EsxDupDstDir = $($EsxDupFolders|Sort-Object CreationTimeUtc -Descending)[0] - - try { - $EsxDupSrcWsp = Get-Item $($EsxDupSrcDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - $EsxDupDstWsp = Get-Item $($EsxDupDstDir.FullName + "/quickstats/Uptime.wsp")|Select-Object BaseName, FullName, CreationTimeUtc, LastWriteTimeUtc, LastAccessTimeUtc - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Missing committed.wsp for Esx $EsxDup ..." - continue - } - - if (($EsxDupDstDir.CreationTimeUtc -gt $EsxDupSrcDir.CreationTimeUtc) -and ($EsxDupDstWsp.LastWriteTimeUtc - $EsxDupSrcWsp.LastWriteTimeUtc).TotalMinutes -gt 90) { - - $EsxDupDstVc = $EsxDupDstDir.FullName.split("/")[5] - $EsxDupDstDc = $EsxDupDstDir.FullName.split("/")[6] - $EsxDupDstClu = $EsxDupDstDir.FullName.split("/")[7] - $EsxDupSrcClu = $EsxDupSrcDir.FullName.split("/")[7] - Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup has been moved from cluster $EsxDupSrcClu to cluster $EsxDupDstClu a while ago, merging metrics to the new destination if possible ..." - $EsxDupWsps2Mv = Get-ChildItem -Recurse $EsxDupSrcDir.FullName -Filter *.wsp - foreach ($EsxDupWsp2Mv in $EsxDupWsps2Mv) { - $WspRelativePath = $($EsxDupWsp2Mv.FullName -split "/Esx/")[1] - $DstWspFullPath = $("/mnt/wfs/whisper/vmw/" + $EsxDupDstVc + "/" + $EsxDupDstDc + "/" + $EsxDupDstClu + "/esx/" + $WspRelativePath) - if (Test-Path $DstWspFullPath) { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $DstWspFullPath whisper-info" - $VmDupWspSrcInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $DstWspFullPath" - Write-Host "$((Get-Date).ToString("o")) [INFO] Checking $($EsxDupWsp2Mv.FullName) whisper-info" - $EsxDupWsp2MvInfo = Invoke-Expression "/usr/local/bin/whisper-info.py $($EsxDupWsp2Mv.FullName)" - - if (Compare-Object $VmDupWspSrcInfo $EsxDupWsp2MvInfo) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Resizing $($EsxDupWsp2Mv.FullName) and $DstWspFullPath" - $EsxDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $EsxDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($EsxDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - } - - Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($EsxDupWsp2Mv.FullName) to $DstWspFullPath" - $EsxDupWsp2MEsxerg = Invoke-Expression "/usr/local/bin/whisper-merge.py $($EsxDupWsp2Mv.FullName) $DstWspFullPath" - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." - continue - } - } else { - try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Creating $DstWspFullPath and moving $($EsxDupWsp2Mv.FullName)" - $EsxDupWspMkDir = New-Item $DstWspFullPath -Force -ErrorAction Stop - $EsxDupWspMv = Move-Item $EsxDupWsp2Mv.FullName $DstWspFullPath -Force -ErrorAction Stop - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($EsxDupWsp2Mv.FullName) moving issue ..." - continue - } - } - } - Write-Host "$((Get-Date).ToString("o")) [INFO] Removing $($EsxDupSrcDir.FullName)" - try { - Remove-Item -Recurse $($EsxDupSrcDir.FullName) -Force -ErrorAction Stop - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Removing $($EsxDupSrcDir.FullName) issue ..." - } - } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] Esx $EsxDup move is too recent, has clones or has come back to its original location ..." - } - } - } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated Esx folders found" - } - } - - Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf has left the building ..." + Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf ViOfflineInventory has left the building ..." } else { AltAndCatchFire "No VI server to process" From 4343a605bd7e9f77683b90f1699d52e94625e359 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 21 Jul 2023 13:57:07 +0200 Subject: [PATCH 093/201] Update xMotionGarbageCollector.ps1 --- opt/sexigraf/xMotionGarbageCollector.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opt/sexigraf/xMotionGarbageCollector.ps1 b/opt/sexigraf/xMotionGarbageCollector.ps1 index a02eac83..f1db23f4 100644 --- a/opt/sexigraf/xMotionGarbageCollector.ps1 +++ b/opt/sexigraf/xMotionGarbageCollector.ps1 @@ -1,7 +1,7 @@ #!/usr/bin/pwsh -Command # -$ScriptVersion = "0.9.1" +$ScriptVersion = "0.9.2" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -70,7 +70,7 @@ if ($VmFolders) { if ($VmFoldersDup_h.Count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] $($VmFoldersDup_h.Count) Duplicate vm folders found across clusters, evaluating mobility ..." - foreach ($VmDup in $($VmFoldersDup_h.keys|Select-Object -first 100)) { + foreach ($VmDup in $($VmFoldersDup_h.keys)) { if ($VmFolders_h[$VmDup].count -lt 2) { Write-Host "$((Get-Date).ToString("o")) [EROR] VM $VmDup has less than 2 copies, skipping ..." @@ -174,7 +174,7 @@ if ($EsxFolders) { if ($EsxFoldersDup_h.Count -gt 0) { Write-Host "$((Get-Date).ToString("o")) [INFO] $($EsxFoldersDup_h.Count) Duplicate Esx folders found across clusters, evaluating mobility ..." - foreach ($EsxDup in $($EsxFoldersDup_h.keys|Select-Object -first 10)) { + foreach ($EsxDup in $($EsxFoldersDup_h.keys)) { if ($EsxFolders_h[$EsxDup].count -lt 2) { Write-Host "$((Get-Date).ToString("o")) [EROR] Esx $EsxDup has less than 2 copies, skipping ..." From 96b2fa6ef1d6a88e8d0397185cc3dcfb1bfe9684 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 23 Jul 2023 16:41:03 +0200 Subject: [PATCH 094/201] Update showlog.php --- var/www/admin/showlog.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/www/admin/showlog.php b/var/www/admin/showlog.php index cae1aa34..8b862123 100644 --- a/var/www/admin/showlog.php +++ b/var/www/admin/showlog.php @@ -259,6 +259,8 @@ function updateLog() { "vSAN Statistics" => "/var/log/sexigraf/VsanDisksPullStatistics.log", "VI Statistics" => "/var/log/sexigraf/ViPullStatistics.log", "VBR Statistics" => "/var/log/sexigraf/VbrPullStatistics.log", + "xMotion Garbage Collector" => "/var/log/sexigraf/xMotionGarbageCollector.log", + "Whisper Auto Purge" => "/var/log/sexigraf/WhisperAutoPurge.log", )); /** From 4c381bd9bc612c8c453b3221f9c64cd52959f394 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 23 Jul 2023 16:41:09 +0200 Subject: [PATCH 095/201] Update purge.php --- var/www/admin/purge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/admin/purge.php b/var/www/admin/purge.php index 6abf63e0..cdf561c9 100644 --- a/var/www/admin/purge.php +++ b/var/www/admin/purge.php @@ -109,7 +109,7 @@
- Force purge deleted items older than days
+ Force purge deleted items older than days
From 7708d291b47bd0a1245667be2fcaa91b35980b89 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 23 Jul 2023 16:41:20 +0200 Subject: [PATCH 096/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 5b7abd69..a6d826b2 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -334,7 +334,11 @@ if ($($VbrJobsStates.data)) { if ($ViVmInventoryTable[$vm_name]) { $vcenter_name = NameCleaner $ViVmInventoryTable[$vm_name].vCenter - $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].Cluster + if ($ViVmInventoryTable[$vm_name].Cluster) { + $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].Cluster + } else { + $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].ESX + } $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.objectRestorePoints"] ++ $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount } From 518cb8ccddf61c4cc08f9624df0c24b8eaed4275 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 23 Jul 2023 16:41:27 +0200 Subject: [PATCH 097/201] Update xMotionGarbageCollector.ps1 --- opt/sexigraf/xMotionGarbageCollector.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opt/sexigraf/xMotionGarbageCollector.ps1 b/opt/sexigraf/xMotionGarbageCollector.ps1 index f1db23f4..7c577804 100644 --- a/opt/sexigraf/xMotionGarbageCollector.ps1 +++ b/opt/sexigraf/xMotionGarbageCollector.ps1 @@ -1,11 +1,12 @@ #!/usr/bin/pwsh -Command # -$ScriptVersion = "0.9.2" +$ScriptVersion = "0.9.3" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" -(Get-Process -Id $pid).PriorityClass = 'BelowNormal' +(Get-Process -Id $pid).PriorityClass = 'idle' +# $stopwatch = [system.diagnostics.stopwatch]::StartNew() function AltAndCatchFire { Param($ExitReason) @@ -140,7 +141,6 @@ if ($VmFolders) { } } else { Write-Host "$((Get-Date).ToString("o")) [INFO] VM $VmDup move is too recent, has clones or has come back to its original location ..." - # TODO deal with infinite clones } } } else { @@ -250,5 +250,5 @@ if ($EsxFolders) { Write-Host "$((Get-Date).ToString("o")) [INFO] No duplicated Esx folders found" } } - +Write-Host "$((Get-Date).ToString("o")) [INFO] If you get of lot of clones in this run, you should consider the Autopurge feature" Write-Host "$((Get-Date).ToString("o")) [INFO] SexiGraf xMotionGarbageCollector has left the building ..." \ No newline at end of file From e5cf5a1722c757769b65c1cdf58ad0d0abc9f341 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 23 Jul 2023 16:42:20 +0200 Subject: [PATCH 098/201] Update xMotionGarbageCollector --- etc/cron.d/xMotionGarbageCollector | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/cron.d/xMotionGarbageCollector b/etc/cron.d/xMotionGarbageCollector index 99f91d18..297c04f8 100644 --- a/etc/cron.d/xMotionGarbageCollector +++ b/etc/cron.d/xMotionGarbageCollector @@ -1 +1 @@ -37 1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/xMotionGarbageCollector.ps1 >/dev/null 2>&1 +37 1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/xMotionGarbageCollector.ps1 >/dev/null 2>&1 \ No newline at end of file From ae4710aed094ad756ddd6c7a8479d9c6d5cfbf3e Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 23 Jul 2023 16:42:46 +0200 Subject: [PATCH 099/201] #284 --- opt/sexigraf/ViPullStatistics.ps1 | 70 ++++---- .../VMware_All_Cluster_Top_N_VM_Stats.json | 75 +++++++++ .../VMware_All_Cluster_VM_Stats.json | 150 ++++++++++++++++++ ...are_All_Standalone_ESX_Top_N_VM_Stats.json | 75 +++++++++ .../VMware_All_Standalone_ESX_VM_Stats.json | 75 +++++++++ 5 files changed, 408 insertions(+), 37 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 45be5732..61697ef0 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1018" +$ScriptVersion = "0.9.1019" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -549,19 +549,17 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { AltAndCatchFire "VM MultiQueryPerf failure" } - # $VmMultiMetricsAll = @( - # "virtualdisk.numberWriteAveraged.average", - # "virtualdisk.numberReadAveraged.average", - # "net.packetsRx.summation", - # "net.packetsTx.summation" - # ) + $VmMultiMetricsAll = @( + "virtualdisk.numberWriteAveraged.average", + "virtualdisk.numberReadAveraged.average", + ) - # try { - # $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} - # Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" - # } catch { - # AltAndCatchFire "VM MultiQueryPerfAll failure" - # } + try { + $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} + Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + } catch { + AltAndCatchFire "VM MultiQueryPerfAll failure" + } } @@ -1059,10 +1057,10 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.diskUsage", 0) } - # if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_cluster_vm.moref.value]) { - # $vcenter_cluster_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.diskIOPS", $vcenter_cluster_vm_disk_iops) - # } + if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_cluster_vm.moref.value]) { + $vcenter_cluster_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.fatstats.diskIOPS", $vcenter_cluster_vm_disk_iops) + } # if ($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_cluster_vm.moref.value]) { # $vcenter_cluster_vm_net_iops = $($($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_cluster_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_cluster_vm.moref.value]).Keys]|Measure-Object -Sum).Sum) / 300 @@ -1889,10 +1887,10 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.diskUsage", 0) } - # if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_standalone_host_vm.moref.value]) { - # $vcenter_standalone_host_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.diskIOPS", $vcenter_standalone_host_vm_disk_iops) - # } + if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_standalone_host_vm.moref.value]) { + $vcenter_standalone_host_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.diskIOPS", $vcenter_standalone_host_vm_disk_iops) + } # if ($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_standalone_host_vm.moref.value]) { # $vcenter_standalone_host_vm_net_iops = $($($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum) / 300 @@ -2104,12 +2102,10 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { "net.usage.average" ) - # $VmMultiMetricsAll = @( - # "virtualdisk.numberWriteAveraged.average", - # "virtualdisk.numberReadAveraged.average", - # "net.packetsRx.summation", - # "net.packetsTx.summation" - # ) + $VmMultiMetricsAll = @( + "virtualdisk.numberWriteAveraged.average", + "virtualdisk.numberReadAveraged.average" + ) if ($vcenter_vms) { try { @@ -2119,12 +2115,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { AltAndCatchFire "VM MultiQueryPerf failure" } - # try { - # $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} - # Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" - # } catch { - # AltAndCatchFire "VM MultiQueryPerfAll failure" - # } + try { + $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} + Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + } catch { + AltAndCatchFire "VM MultiQueryPerfAll failure" + } } try { @@ -2444,10 +2440,10 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.diskUsage", 0) } - # if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$unmanaged_host_vm.moref.value]) { - # $unmanaged_host_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum - # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.diskIOPS", $unmanaged_host_vm_disk_iops) - # } + if ($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$unmanaged_host_vm.moref.value]) { + $unmanaged_host_vm_disk_iops = $($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["virtualdisk.numberReadAveraged.average"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["virtualdisk.numberWriteAveraged.average"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.diskIOPS", $unmanaged_host_vm_disk_iops) + } # if ($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$unmanaged_host_vm.moref.value]) { # $unmanaged_host_vm_net_iops = $($($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$unmanaged_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$unmanaged_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum) / 300 diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json index b03256ff..ef78bd61 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json @@ -614,6 +614,81 @@ "show": true } ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "id": 14, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$cluster.vm.$vm.fatstats.diskiops, '$N')), 5)", + "textEditor": true + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } ], "repeat": null, diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json index 07d12a13..b3c7912f 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json @@ -614,6 +614,156 @@ "show": true } ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "id": 14, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.vm.$vm.fatstats.diskiops, 5)", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "id": 15, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "sortByMaxima(aliasByNode(veeam.vi.$vcenter.$cluster.vm.$vm.restorePointsCount, 5))", + "textEditor": false + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Veeam Backup Restore Points", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } ], "repeat": null, diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json index cc01e4bd..eb80ef8a 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json @@ -614,6 +614,81 @@ "show": true } ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "id": 14, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskiops, '$N')), 5)", + "textEditor": true + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } ], "repeat": null, diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json index c203877a..c04f0bf3 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json @@ -614,6 +614,81 @@ "show": true } ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "id": 14, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "refId": "A", + "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskiops, '$N')), 5)", + "textEditor": true + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } ], "repeat": null, From a2fccf3ee163dc7037ac96271757896b06a45eff Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 23 Jul 2023 19:16:18 +0200 Subject: [PATCH 100/201] #334 --- opt/sexigraf/ViPullStatistics.ps1 | 11 +- .../VMware_Multi_Datastore_Usage_Pct.json | 251 ++++++++++++++++++ 2 files changed, 259 insertions(+), 3 deletions(-) create mode 100644 var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Pct.json diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 61697ef0..1dc0829b 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1019" +$ScriptVersion = "0.9.1020" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -551,7 +551,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $VmMultiMetricsAll = @( "virtualdisk.numberWriteAveraged.average", - "virtualdisk.numberReadAveraged.average", + "virtualdisk.numberReadAveraged.average" ) try { @@ -1126,7 +1126,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_datastore_hosts = ($vcenter_cluster_datastore.Host|?{$_.mountinfo.Accessible}).key - if($vcenter_cluster_datastore.summary.uncommitted -ge 0) { + if ($vcenter_cluster_datastore.summary.uncommitted -ge 0) { $vcenter_cluster_datastore_uncommitted = $vcenter_cluster_datastore.summary.uncommitted } else { $vcenter_cluster_datastore_uncommitted = 0 @@ -1136,6 +1136,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.summary.freeSpace", $vcenter_cluster_datastore.summary.freeSpace) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.summary.uncommitted", $vcenter_cluster_datastore_uncommitted) + if ($vcenter_cluster_datastore.summary.capacity -gt 0 -and $vcenter_cluster_datastore.summary.capacity -gt $vcenter_cluster_datastore.summary.freeSpace) { + $vcenter_cluster_datastore_usagepct = ($vcenter_cluster_datastore.summary.capacity - $vcenter_cluster_datastore.summary.freeSpace) * 100 / $vcenter_cluster_datastore.summary.capacity + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.summary.usagePct", $vcenter_cluster_datastore_usagepct) + } + if ($vcenter_cluster_vmdk_per_ds[$vcenter_cluster_datastore_name]) { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.summary.vmdkCount", $vcenter_cluster_vmdk_per_ds[$vcenter_cluster_datastore_name]) } diff --git a/var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Pct.json b/var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Pct.json new file mode 100644 index 00000000..e407441c --- /dev/null +++ b/var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Pct.json @@ -0,0 +1,251 @@ +{ + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "iteration": 1587995684562, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "repeat": null, + "title": "", + "type": "row" + }, + { + "datasource": null, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 8, + "maxPerRow": 6, + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "max": 104, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(31, 255, 0)", + "value": null + }, + { + "color": "rgb(255, 239, 0)", + "value": 70 + }, + { + "color": "rgb(255, 115, 0)", + "value": 80 + }, + { + "color": "rgb(255, 0, 29)", + "value": 90 + }, + { + "color": "rgb(204, 0, 255)", + "value": 99 + } + ] + }, + "unit": "percent" + }, + "overrides": [], + "values": false + }, + "orientation": "auto", + "showThresholdLabels": false, + "showThresholdMarkers": false + }, + "pluginVersion": "6.7.1", + "repeat": "datastore", + "repeatDirection": "h", + "targets": [ + { + "refCount": 0, + "refId": "B", + "target": "alias(removeEmptySeries(vmw.$vcenter.$datacentre.$cluster.datastore.$datastore.summary.usagePct), 'usagePct')", + "textEditor": false + } + ], + "timeFrom": null, + "timeShift": null, + "title": "$datastore", + "type": "bargauge" + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "light", + "tags": [ + "HDD" + ], + "templating": { + "list": [ + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": "local", + "definition": "vmw.*", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "vcenter", + "options": [], + "query": "vmw.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": "local", + "definition": "vmw.$vcenter.*", + "hide": 0, + "includeAll": true, + "index": -1, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "datacentre", + "options": [], + "query": "vmw.$vcenter.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": "local", + "definition": "vmw.$vcenter.$datacentre.*", + "hide": 0, + "includeAll": true, + "index": -1, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "cluster", + "options": [], + "query": "vmw.$vcenter.$datacentre.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": "local", + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "datastore", + "options": [], + "query": "vmw.$vcenter.$datacentre.$cluster.datastore.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "15m", + "1h", + "2h", + "3h", + "6h", + "12h", + "24h", + "2d", + "3d", + "7d", + "15d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "VMware Multi Datastore Usage %", + "uid": "6e3fb4e8", + "variables": { + "list": [] + }, + "version": 14 +} \ No newline at end of file From 1f1a14b784ff08e522ac229a47dd74803c9ff7c8 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Mon, 24 Jul 2023 18:46:06 +0200 Subject: [PATCH 101/201] Update VMware_All_Cluster_SuperStats.json #258 --- .../VMware_All_Cluster_SuperStats.json | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json index 3bd22909..cdf4b8e8 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json @@ -1352,6 +1352,190 @@ "alignLevel": null } }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 69, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dead" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "disabled" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "standby" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 18, + "links": [], + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "A", + "target": "sortBy(groupByNode(removeEmptySeries(vmw.$vcenter.$datacentre.$cluster.superstats.PathState.*), 6, 'total'), 'last', false)", + "textEditor": false + } + ], + "title": "pathState count", + "type": "timeseries" + }, { "collapsed": false, "datasource": null, From 0891b18c314c55be6ab96dd904e315c887a63031 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Mon, 24 Jul 2023 19:20:13 +0200 Subject: [PATCH 102/201] Update VMware_All_Cluster_SuperStats.json --- .../VMware_Cluster/VMware_All_Cluster_SuperStats.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json index cdf4b8e8..20af7029 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_SuperStats.json @@ -1533,7 +1533,7 @@ "textEditor": false } ], - "title": "pathState count", + "title": "MultipathState Count", "type": "timeseries" }, { From 5d1dbb73d4112292a455819ca4eb16598152fbe9 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Mon, 24 Jul 2023 19:20:31 +0200 Subject: [PATCH 103/201] Update VMware_Cluster_FullStats.json #258 --- .../VMware_Cluster_FullStats.json | 350 +++++++++++++----- 1 file changed, 267 insertions(+), 83 deletions(-) diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json index 1c5d4498..25c1f350 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json @@ -1307,40 +1307,36 @@ } }, { - "aliasColors": { - "svmotion": "dark-orange", - "vmotion": "#052B51", - "xvmotion": "super-light-blue" - }, - "bars": true, + "aliasColors": {}, + "bars": false, "dashLength": 10, "dashes": false, "datasource": "local", - "decimals": 0, + "decimals": 1, "editable": false, "error": false, - "fill": 4, + "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 6, - "w": 6, + "w": 12, "x": 0, "y": 31 }, "hiddenSeries": false, - "id": 4, + "id": 13, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": true, - "total": true, - "values": true + "show": false, + "total": false, + "values": false }, - "lines": false, - "linewidth": 2, + "lines": true, + "linewidth": 3, "links": [], "nullPointMode": "null", "options": { @@ -1351,36 +1347,39 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "/vCPU/i", + "color": "#6D1F62" + } + ], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { + "hide": true, "refId": "A", - "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numVmotions), 'vmotion')", - "textEditor": false + "target": "vmw.$vcenter.$datacentre.$cluster.quickstats.pCPUs" }, { - "refCount": 0, + "hide": true, "refId": "B", - "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numSVMotions), 'svmotion')", - "textEditor": false + "target": "vmw.$vcenter.$datacentre.$cluster.quickstats.vCPUs" }, { - "refCount": 0, "refId": "C", - "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numXVMotions), 'xvmotion')", + "target": "alias(divideSeries(#B, #A), '$cluster.vCPU/pCPU')", "textEditor": false } ], "thresholds": [], "timeRegions": [], - "title": "vMotion count", + "title": "vCPU per pCPU ratio", "tooltip": { "shared": true, "sort": 2, - "value_type": "individual" + "value_type": "cumulative" }, "type": "graph", "xaxis": { @@ -1390,7 +1389,7 @@ }, "yaxes": [ { - "format": "none", + "format": "short", "logBase": 1, "min": 0, "show": true @@ -1411,7 +1410,6 @@ "dashLength": 10, "dashes": false, "datasource": "local", - "decimals": 1, "editable": false, "error": false, "fill": 0, @@ -1420,11 +1418,11 @@ "gridPos": { "h": 6, "w": 6, - "x": 6, + "x": 12, "y": 31 }, "hiddenSeries": false, - "id": 13, + "id": 9, "legend": { "avg": false, "current": false, @@ -1435,7 +1433,7 @@ "values": false }, "lines": true, - "linewidth": 3, + "linewidth": 2, "links": [], "nullPointMode": "null", "options": { @@ -1446,35 +1444,20 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/vCPU/i", - "color": "#6D1F62" - } - ], + "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "hide": true, + "hide": false, "refId": "A", - "target": "vmw.$vcenter.$datacentre.$cluster.quickstats.pCPUs" - }, - { - "hide": true, - "refId": "B", - "target": "vmw.$vcenter.$datacentre.$cluster.quickstats.vCPUs" - }, - { - "refId": "C", - "target": "alias(divideSeries(#B, #A), '$cluster.vCPU/pCPU')", - "textEditor": false + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.esx.*.quickstats.distributedCpuFairness, 5)" } ], "thresholds": [], "timeRegions": [], - "title": "vCPU per pCPU ratio", + "title": "distributed CPU fairness", "tooltip": { "shared": true, "sort": 2, @@ -1488,12 +1471,13 @@ }, "yaxes": [ { + "$$hashKey": "object:1117", "format": "short", "logBase": 1, - "min": 0, "show": true }, { + "$$hashKey": "object:1118", "format": "short", "logBase": 1, "show": true @@ -1517,11 +1501,11 @@ "gridPos": { "h": 6, "w": 6, - "x": 12, + "x": 18, "y": 31 }, "hiddenSeries": false, - "id": 9, + "id": 18, "legend": { "avg": false, "current": false, @@ -1549,14 +1533,13 @@ "steppedLine": false, "targets": [ { - "hide": false, - "refId": "A", - "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.esx.*.quickstats.distributedCpuFairness, 5)" + "refId": "B", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.esx.*.quickstats.distributedMemoryFairness, 5)" } ], "thresholds": [], "timeRegions": [], - "title": "distributed CPU fairness", + "title": "distributed RAM fairness", "tooltip": { "shared": true, "sort": 2, @@ -1570,16 +1553,17 @@ }, "yaxes": [ { - "$$hashKey": "object:1117", + "$$hashKey": "object:1279", "format": "short", "logBase": 1, + "min": "0", "show": true }, { - "$$hashKey": "object:1118", + "$$hashKey": "object:1280", "format": "short", "logBase": 1, - "show": true + "show": false } ], "yaxis": { @@ -1587,34 +1571,39 @@ } }, { - "aliasColors": {}, - "bars": false, + "aliasColors": { + "svmotion": "dark-orange", + "vmotion": "#052B51", + "xvmotion": "super-light-blue" + }, + "bars": true, "dashLength": 10, "dashes": false, "datasource": "local", + "decimals": 0, "editable": false, "error": false, - "fill": 0, + "fill": 4, "fillGradient": 0, "grid": {}, "gridPos": { "h": 6, - "w": 6, - "x": 18, - "y": 31 + "w": 12, + "x": 0, + "y": 37 }, "hiddenSeries": false, - "id": 18, + "id": 4, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, - "total": false, - "values": false + "show": true, + "total": true, + "values": true }, - "lines": true, + "lines": false, "linewidth": 2, "links": [], "nullPointMode": "null", @@ -1628,21 +1617,34 @@ "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { + "refId": "A", + "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numVmotions), 'vmotion')", + "textEditor": false + }, + { + "refCount": 0, "refId": "B", - "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.esx.*.quickstats.distributedMemoryFairness, 5)" + "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numSVMotions), 'svmotion')", + "textEditor": false + }, + { + "refCount": 0, + "refId": "C", + "target": "alias(nonNegativeDerivative(vmw.$vcenter.$datacentre.$cluster.quickstats.numXVMotions), 'xvmotion')", + "textEditor": false } ], "thresholds": [], "timeRegions": [], - "title": "distributed RAM fairness", + "title": "vMotion count", "tooltip": { "shared": true, "sort": 2, - "value_type": "cumulative" + "value_type": "individual" }, "type": "graph", "xaxis": { @@ -1652,23 +1654,205 @@ }, "yaxes": [ { - "$$hashKey": "object:1279", - "format": "short", + "format": "none", "logBase": 1, - "min": "0", + "min": 0, "show": true }, { - "$$hashKey": "object:1280", "format": "short", "logBase": 1, - "show": false + "show": true } ], "yaxis": { "align": false } }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 69, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "dead" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "disabled" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#3274D9", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "standby" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2CC0C", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FF780A", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 19, + "links": [], + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": "local", + "hide": false, + "refCount": 0, + "refId": "A", + "target": "sortBy(groupByNode(removeEmptySeries(vmw.$vcenter.$datacentre.$cluster.superstats.PathState.*), 6, 'total'), 'last', false)", + "textEditor": false + } + ], + "title": "MultipathState Count", + "type": "timeseries" + }, { "collapsed": false, "datasource": "local", @@ -1676,7 +1860,7 @@ "h": 1, "w": 24, "x": 0, - "y": 37 + "y": 43 }, "id": 17, "panels": [], @@ -1698,7 +1882,7 @@ "h": 6, "w": 12, "x": 0, - "y": 38 + "y": 44 }, "hiddenSeries": false, "id": 11, @@ -1795,7 +1979,7 @@ "h": 6, "w": 12, "x": 12, - "y": 38 + "y": 44 }, "hiddenSeries": false, "id": 12, From 8c900b8ad5a4deb6f6072fa7c7c5e7663b62f103 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Mon, 24 Jul 2023 22:56:42 +0200 Subject: [PATCH 104/201] Update purge.php --- var/www/admin/purge.php | 1 + 1 file changed, 1 insertion(+) diff --git a/var/www/admin/purge.php b/var/www/admin/purge.php index cdf561c9..81bcee37 100644 --- a/var/www/admin/purge.php +++ b/var/www/admin/purge.php @@ -11,6 +11,7 @@
  • This page can be used to purge old and/or unwanted whisper data objects as well as vCenter session files.
  • Beware as this operation cannot be undone, so there is a risk of DATA LOSS if you don't know what you're doing.
  • Autopurge will automatically remove all files not updated since selected days (i.e. deleted vms).
    • +
  • You can check the WhisperAutoPurge log in the Log Viewer panel to follow the process.
  • Please refer to the project website and documentation for more information.
    • From 57e484209f05bd3432de0dcebfefef40128dfde5 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Tue, 25 Jul 2023 14:58:09 +0200 Subject: [PATCH 105/201] Update Veeam_Restore_Point.json --- var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json index a49612a8..ffc10f5c 100644 --- a/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json @@ -134,7 +134,7 @@ { "datasource": "local", "refId": "A", - "target": "groupByNode(removeEmptySeries(veeam.vi.$vcenter.$cluster.vm.$vm.objectRestorePoints), 3, 'total')" + "target": "groupByNode(removeEmptySeries(veeam.vi.$vcenter.$cluster.vm.*.objectRestorePoints), 3, 'total')" } ], "thresholds": [], From b3e99dcb63d4ec39e9182c89c21cd71c8ecc2bc7 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 11 Aug 2023 14:28:22 +0200 Subject: [PATCH 106/201] #351 --- opt/graphite/conf/storage-schemas.conf | 10 +++++----- opt/sexigraf/xMotionGarbageCollector.ps1 | 8 ++++---- var/www/scripts/importSexiGrafBundle.sh | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/opt/graphite/conf/storage-schemas.conf b/opt/graphite/conf/storage-schemas.conf index d04ff74e..ed2f8d00 100644 --- a/opt/graphite/conf/storage-schemas.conf +++ b/opt/graphite/conf/storage-schemas.conf @@ -38,19 +38,19 @@ retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y [vsan_sync] pattern = .*\.vsan\.SyncingVsanObjects\. -retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d +retentions = 5m:24h,10m:48h,30m:96h,60m:7d,240m:30d,720m:90d [vsan_smart] pattern = .*\.vsan\.disk\.smart\. -retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y +retentions = 5m:24h,10m:48h,30m:96h,60m:7d,240m:30d,720m:90d,2880m:1y [vsan_spaceDetail] pattern = .*\.vsan\.spaceDetail\. -retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y +retentions = 5m:24h,10m:48h,30m:96h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y [vsan] pattern = ^vsan\. -retentions = 5m:12h,10m:24h,20m:48h,80m:7d,320m:30d,640m:90d +retentions = 5m:12h,10m:24h,20m:48h,40m:96h,80m:7d,320m:30d,640m:90d [veeam] pattern = ^veeam\. @@ -58,4 +58,4 @@ retentions = 5m:7d,10m:30d,15m:90d,30m:120d,120m:1y [anything_else] pattern = .* -retentions = 5m:24h,10m:48h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y \ No newline at end of file +retentions = 5m:24h,10m:48h,30m:96h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y \ No newline at end of file diff --git a/opt/sexigraf/xMotionGarbageCollector.ps1 b/opt/sexigraf/xMotionGarbageCollector.ps1 index 7c577804..9736aba0 100644 --- a/opt/sexigraf/xMotionGarbageCollector.ps1 +++ b/opt/sexigraf/xMotionGarbageCollector.ps1 @@ -112,8 +112,8 @@ if ($VmFolders) { if (Compare-Object $VmDupWspSrcInfo $VmDupWsp2MvInfo) { Write-Host "$((Get-Date).ToString("o")) [INFO] Resizing $($VmDupWsp2Mv.FullName) and $DstWspFullPath" - $VmDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $VmDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($VmDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $VmDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $VmDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($VmDupWsp2Mv.FullName) 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" } Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($VmDupWsp2Mv.FullName) to $DstWspFullPath" @@ -215,8 +215,8 @@ if ($EsxFolders) { if (Compare-Object $VmDupWspSrcInfo $EsxDupWsp2MvInfo) { Write-Host "$((Get-Date).ToString("o")) [INFO] Resizing $($EsxDupWsp2Mv.FullName) and $DstWspFullPath" - $EsxDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" - $EsxDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($EsxDupWsp2Mv.FullName) 5m:24h 10m:48h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $EsxDupWspSrcResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $DstWspFullPath 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" + $EsxDupWsp2MvResiz = Invoke-Expression "/usr/local/bin/whisper-resize.py $($EsxDupWsp2Mv.FullName) 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup --force" } Write-Host "$((Get-Date).ToString("o")) [INFO] Merging $($EsxDupWsp2Mv.FullName) to $DstWspFullPath" diff --git a/var/www/scripts/importSexiGrafBundle.sh b/var/www/scripts/importSexiGrafBundle.sh index dc4f885d..dfc155ae 100644 --- a/var/www/scripts/importSexiGrafBundle.sh +++ b/var/www/scripts/importSexiGrafBundle.sh @@ -86,9 +86,9 @@ if [ -d "/mnt/wfs/whisper" ]; then /etc/init.d/grafana-server restart apachectl restart - # 0.99g storage schema change #217 - # nohup /usr/bin/find -L /mnt/wfs/whisper/vmw/ -type f \( -name '*.wsp' \) -exec /usr/local/bin/whisper-resize.py {} 5m:24h 10m:48h 80m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup \; > /var/log/sexigraf/vmw_wsp_resize.log 2>&1 & - # nohup /usr/bin/find -L /mnt/wfs/whisper/esx/ -type f \( -name '*.wsp' \) -exec /usr/local/bin/whisper-resize.py {} 5m:24h 10m:48h 80m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup \; > /var/log/sexigraf/esx_wsp_resize.log 2>&1 & + # 0.99j storage schema change #351 + # nohup /usr/bin/find -L /mnt/wfs/whisper/vmw/ -type f \( -name '*.wsp' \) -exec /usr/local/bin/whisper-resize.py {} 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup \; > /var/log/sexigraf/vmw_wsp_resize.log 2>&1 & + # nohup /usr/bin/find -L /mnt/wfs/whisper/esx/ -type f \( -name '*.wsp' \) -exec /usr/local/bin/whisper-resize.py {} 5m:24h 10m:48h 30m:96h 60m:7d 240m:30d 720m:90d 2880m:1y 5760m:2y 17280m:5y --nobackup \; > /var/log/sexigraf/esx_wsp_resize.log 2>&1 & # Inventory Update nohup /bin/bash /var/www/scripts/updateInventory.sh & From a21238d0504bd59ab5ef9af185b742907c296615 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 11 Aug 2023 14:28:29 +0200 Subject: [PATCH 107/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index a6d826b2..02a4a1c3 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.24" +$ScriptVersion = "0.9.25" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -256,7 +256,7 @@ if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } - $VbrVmwareServers = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/inventory/vmware/hosts") -Headers $VbrAuthHeaders + # $VbrVmwareServers = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/inventory/vmware/hosts") -Headers $VbrAuthHeaders if ($VbrObjectRestorePoints5.data) { From cc13f0bb6de5defe8635da88050d99247dcc3e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sun, 3 Sep 2023 23:32:01 +0200 Subject: [PATCH 108/201] Create VbrVmInventory.csv #147 --- opt/sexigraf/VbrVmInventory.csv | 1 + 1 file changed, 1 insertion(+) create mode 100644 opt/sexigraf/VbrVmInventory.csv diff --git a/opt/sexigraf/VbrVmInventory.csv b/opt/sexigraf/VbrVmInventory.csv new file mode 100644 index 00000000..b6c16e1d --- /dev/null +++ b/opt/sexigraf/VbrVmInventory.csv @@ -0,0 +1 @@ +"VbrServer", "JobName", "vCenter", "Cluster", "VM", "RestorePointsCount", "LastRestorePoint" \ No newline at end of file From 3fb56e5bcb203dd35b03ecfe1374a669a76d8ba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sun, 3 Sep 2023 23:32:11 +0200 Subject: [PATCH 109/201] Update VbrPullStatistics.ps1 #147 --- opt/veeam/VbrPullStatistics.ps1 | 111 +++++++++++++++++++++++--------- 1 file changed, 79 insertions(+), 32 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 02a4a1c3..5a2b12ac 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.25" +$ScriptVersion = "0.9.30" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -223,8 +223,8 @@ if ($($VbrJobsStates.data)) { if ($VbrSobrNotNormal.id) { $VbrSobrNotNormalRepos = $VbrRepositoryTable[$VbrSobrNotNormal.id] - $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.capacityGB"] + $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property capacityGB).Sum - $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] + $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property usedSpaceGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.capacityGB"] += $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property capacityGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] += $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property usedSpaceGB).Sum } $VbrSobrNormal = $VbrScaleOutRepository.performanceTier.performanceExtents|?{$_.status -eq "Normal"} @@ -250,7 +250,7 @@ if ($($VbrJobsStates.data)) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] VBR 5min old objectRestorePoints collect ..." # $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders - $VbrObjectRestorePoints5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/objectRestorePoints?platformNameFilter=VmWare&createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + $VbrObjectRestorePoints5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/objectRestorePoints?platformNameFilter=VmWare&limit=999&createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] objectRestorePoints collect failure" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -307,45 +307,92 @@ if ($($VbrJobsStates.data)) { } } - foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { - $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name - $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ + if (Test-Path /mnt/wfs/inventory/ViVmInventory.csv) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Importing VM inventory ..." + try { + $ViVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop) + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] ViVmInventory import issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } - if (Test-Path /mnt/wfs/inventory/ViVmInventory.csv) { - try { - $ViVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop) - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ViVmInventory import issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + if ($ViVmInventory) { + $ViVmInventoryTable = @{} + foreach ($ViVm in $ViVmInventory) { + try { + $ViVmInventoryTable.Add($ViVm.vm,$ViVm) + } catch {} } - if ($ViVmInventory) { - $ViVmInventoryTable = @{} - foreach ($ViVm in $ViVmInventory) { - try { - $ViVmInventoryTable.Add($ViVm.vm,$ViVm) - } catch {} + if (Test-Path /mnt/wfs/inventory/VbrInv_$($Server).csv) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Importing $($Server) VBR inventory ..." + try { + $VbrVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/VbrInv_$($Server).csv -ErrorAction Stop) + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] VbrVmInventory import issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + if ($ViVmInventory) { + $VbrVmInventoryTable = @{} + foreach ($VbrVm in $VbrVmInventory) { + try { + if ($ViVmInventoryTable[$VbrVm.vm]) { + $VbrVmInventoryTable.Add($VbrVm.vm,$VbrVm) + } + } catch {} + } } } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] ViVmInventory import issue" + $VbrVmInventoryTable = @{} } - $vm_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name + Write-Host "$((Get-Date).ToString("o")) [INFO] Building $($Server) VBR DataTable and inventory ..." + foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { + $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name + $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ + $vm_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name + if ($ViVmInventoryTable[$vm_name]) { + $vcenter_name = NameCleaner $ViVmInventoryTable[$vm_name].vCenter + if ($ViVmInventoryTable[$vm_name].Cluster) { + $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].Cluster + } else { + $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].ESX + } + $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.objectRestorePoints"] ++ + $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount + + $VbrObjectInventoryInfo = "" | Select-Object VbrServer, JobName, vCenter, Cluster, VM, RestorePointsCount, LastRestorePoint + $VbrObjectInventoryInfo.VbrServer = $vbrserver_name + $VbrObjectInventoryInfo.JobName = $job_name + $VbrObjectInventoryInfo.vCenter = $vcenter_name + $VbrObjectInventoryInfo.Cluster = $cluster_name + $VbrObjectInventoryInfo.VM = $vm_name + $VbrObjectInventoryInfo.RestorePointsCount = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount + $VbrObjectInventoryInfo.LastRestorePoint = $VbrObjectRestorePoint.creationTime + if ($VbrVmInventoryTable[$vm_name]) { + $VbrVmInventoryTable[$vm_name] = $VbrObjectInventoryInfo + } else { + $VbrVmInventoryTable.add($vm_name,$VbrObjectInventoryInfo) + } + } + } - if ($ViVmInventoryTable[$vm_name]) { - $vcenter_name = NameCleaner $ViVmInventoryTable[$vm_name].vCenter - if ($ViVmInventoryTable[$vm_name].Cluster) { - $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].Cluster - } else { - $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].ESX + if ($VbrVmInventoryTable.Values) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Exporting $($Server) VBR inventory ..." + try { + $VbrVmInventoryTable.Values|Sort-Object VM|Export-Csv /mnt/wfs/inventory/VbrInv_$($Server).csv -ErrorAction Stop -Force + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] VbrVmInventory import issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } - $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.objectRestorePoints"] ++ - $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount + } else { + Write-Host "$((Get-Date).ToString("o")) [EROR] No VbrVmInventoryTable values ?!" } - } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] No ViVmInventory" + Write-Host "$((Get-Date).ToString("o")) [EROR] ViVmInventory import issue" } + } else { + Write-Host "$((Get-Date).ToString("o")) [EROR] No ViVmInventory" } } @@ -376,7 +423,7 @@ if ($($VbrJobsStates.data)) { $VbrDataTable["veeam.vbr.$vbrserver_name.exec.duration"] = $($(Get-Date).ToUniversalTime() - $ExecStart).TotalSeconds Write-Host "$((Get-Date).ToString("o")) [INFO] Sending veeam data to Graphite for VBR server $Server ..." - Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrDataTable -DateTime $ExecStart + # Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrDataTable -DateTime $ExecStart Write-Host "$((Get-Date).ToString("o")) [INFO] End of VBR server $server processing ..." From 087f0fcc537415e8bfed7354381c36ae71c356df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sun, 3 Sep 2023 23:32:16 +0200 Subject: [PATCH 110/201] Create VbrVmInventory.ps1 #147 --- opt/veeam/VbrVmInventory.ps1 | 67 ++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 opt/veeam/VbrVmInventory.ps1 diff --git a/opt/veeam/VbrVmInventory.ps1 b/opt/veeam/VbrVmInventory.ps1 new file mode 100644 index 00000000..3e2774cc --- /dev/null +++ b/opt/veeam/VbrVmInventory.ps1 @@ -0,0 +1,67 @@ +#!/usr/bin/pwsh -Command +# +$ScriptVersion = "0.9.2" + +$ExecStart = $(Get-Date).ToUniversalTime() +# $stopwatch = [system.diagnostics.stopwatch]::StartNew() + +$ErrorActionPreference = "SilentlyContinue" +$WarningPreference = "SilentlyContinue" +(Get-Process -Id $pid).PriorityClass = 'Idle' + +function AltAndCatchFire { + Param($ExitReason) + Write-Host "$((Get-Date).ToString("o")) [EROR] $ExitReason" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + Stop-Transcript + exit +} + +try { + Start-Transcript -Path "/var/log/sexigraf/VbrVmInventory.log" -Append -Confirm:$false -Force -UseMinimalHeader + Write-Host "$((Get-Date).ToString("o")) [INFO] VbrVmInventory v$ScriptVersion" +} catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] VbrVmInventory logging failure" + Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + exit +} + +try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for another VbrVmInventory ..." + $DupVbrVmInventoryProcess = Get-PSHostProcessInfo|%{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}|?{$_ -match "VbrVmInventory"} + # https://github.com/PowerShell/PowerShell/issues/13944 + if (($DupVbrVmInventoryProcess|Measure-Object).Count -gt 1) { + $DupVbrVmInventoryProcessId = (Get-PSHostProcessInfo|?{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}).ProcessId[0] + $DupVbrVmInventoryProcessTime = [INT32](ps -p $DupVbrVmInventoryProcessId -o etimes).split()[-1] + if ($DupVbrVmInventoryProcessTime -gt 300) { + Write-Host "$((Get-Date).ToString("o")) [WARN] VbrVmInventory is already running for more than 5 minutes!" + Write-Host "$((Get-Date).ToString("o")) [WARN] Killing stunned VbrVmInventory" + Stop-Process -Id $DupVbrVmInventoryProcessId -Force + } else { + AltAndCatchFire "VbrVmInventory is already running!" + } + } +} catch { + AltAndCatchFire "VbrVmInventory process lookup failure" +} + +try { + Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for VBR inventories ..." + $VbrVmInventories = Test-Path /mnt/wfs/inventory/VbrInv_*.csv +} catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] No VBR VM Inventories ?!" + Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + exit +} + +if ($VbrVmInventories) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Import/Export VBR inventories ..." + try { + Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force #TODO sort LastRestorePoint + } catch { + AltAndCatchFire "Import/Export VBR inventories issue" + } +} + +Write-Host "$((Get-Date).ToString("o")) [INFO] End of VBR inventories processing ..." \ No newline at end of file From 33858c8f83373567735acfbedc7235f39bd48a72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sun, 3 Sep 2023 23:32:21 +0200 Subject: [PATCH 111/201] Update VMware_Cluster_FullStats.json --- .../dashboards/VMware_Cluster/VMware_Cluster_FullStats.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json index 25c1f350..fa073418 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Cluster_FullStats.json @@ -1824,7 +1824,7 @@ "x": 12, "y": 37 }, - "id": 19, + "id": 20, "links": [], "options": { "legend": { From 9c22a241b9845d068f8c2c6d5118b453f9818740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sun, 3 Sep 2023 23:33:47 +0200 Subject: [PATCH 112/201] Create VMware_VBR_Inventory.json #147 --- .../VMware_VBR_Inventory.json | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json diff --git a/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json new file mode 100644 index 00000000..872d312d --- /dev/null +++ b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json @@ -0,0 +1,90 @@ +{ + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "ViVbrCsv", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 32, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "showHeader": true + }, + "pluginVersion": "7.5.11", + "targets": [ + { + "decimalSeparator": ".", + "delimiter": ",", + "experimental": { + "regex": true + }, + "header": true, + "ignoreUnknown": false, + "refId": "A", + "schema": [ + { + "name": "", + "type": "string" + } + ], + "skipRows": 0 + } + ], + "title": " ", + "type": "table" + } + ], + "schemaVersion": 27, + "style": "dark", + "tags": [ + "VI", + "VM" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "hidden": true + }, + "timezone": "", + "title": "VMware VBR Inventory", + "uid": "iQlLNW57b", + "version": 8 +} \ No newline at end of file From 0ee9dcd99cf41c5eeb4d2740edad77030e7a1393 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sun, 3 Sep 2023 23:38:07 +0200 Subject: [PATCH 113/201] Update addVbrCrontab.sh #147 --- var/www/scripts/addVbrCrontab.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/var/www/scripts/addVbrCrontab.sh b/var/www/scripts/addVbrCrontab.sh index 73528525..5535158e 100644 --- a/var/www/scripts/addVbrCrontab.sh +++ b/var/www/scripts/addVbrCrontab.sh @@ -2,4 +2,5 @@ sessionFile="/tmp/vbr_$(sed s/\\./_/g <<<$1).key" crontabFile="/etc/cron.d/vbr_$(sed s/\\./_/g <<<$1)" echo "*/5 * * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/veeam/VbrPullStatistics.ps1 -credstore /mnt/wfs/inventory/vbrpscredentials.xml -server $1 -sessionfile $sessionFile >/dev/null 2>&1" >> $crontabFile +echo "3 */1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/veeam/VbrVmInventory.ps1 >/dev/null 2>&1" > "/etc/cron.d/VbrVmInventory" service cron reload \ No newline at end of file From 0d92d122e02725ddfb786639fe1ceab20e8ffbb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Mon, 4 Sep 2023 22:54:49 +0200 Subject: [PATCH 114/201] Delete VbrVmInventory.ps1 --- opt/veeam/VbrVmInventory.ps1 | 67 ------------------------------------ 1 file changed, 67 deletions(-) delete mode 100644 opt/veeam/VbrVmInventory.ps1 diff --git a/opt/veeam/VbrVmInventory.ps1 b/opt/veeam/VbrVmInventory.ps1 deleted file mode 100644 index 3e2774cc..00000000 --- a/opt/veeam/VbrVmInventory.ps1 +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/pwsh -Command -# -$ScriptVersion = "0.9.2" - -$ExecStart = $(Get-Date).ToUniversalTime() -# $stopwatch = [system.diagnostics.stopwatch]::StartNew() - -$ErrorActionPreference = "SilentlyContinue" -$WarningPreference = "SilentlyContinue" -(Get-Process -Id $pid).PriorityClass = 'Idle' - -function AltAndCatchFire { - Param($ExitReason) - Write-Host "$((Get-Date).ToString("o")) [EROR] $ExitReason" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" - Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" - Stop-Transcript - exit -} - -try { - Start-Transcript -Path "/var/log/sexigraf/VbrVmInventory.log" -Append -Confirm:$false -Force -UseMinimalHeader - Write-Host "$((Get-Date).ToString("o")) [INFO] VbrVmInventory v$ScriptVersion" -} catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VbrVmInventory logging failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" - exit -} - -try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for another VbrVmInventory ..." - $DupVbrVmInventoryProcess = Get-PSHostProcessInfo|%{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}|?{$_ -match "VbrVmInventory"} - # https://github.com/PowerShell/PowerShell/issues/13944 - if (($DupVbrVmInventoryProcess|Measure-Object).Count -gt 1) { - $DupVbrVmInventoryProcessId = (Get-PSHostProcessInfo|?{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}).ProcessId[0] - $DupVbrVmInventoryProcessTime = [INT32](ps -p $DupVbrVmInventoryProcessId -o etimes).split()[-1] - if ($DupVbrVmInventoryProcessTime -gt 300) { - Write-Host "$((Get-Date).ToString("o")) [WARN] VbrVmInventory is already running for more than 5 minutes!" - Write-Host "$((Get-Date).ToString("o")) [WARN] Killing stunned VbrVmInventory" - Stop-Process -Id $DupVbrVmInventoryProcessId -Force - } else { - AltAndCatchFire "VbrVmInventory is already running!" - } - } -} catch { - AltAndCatchFire "VbrVmInventory process lookup failure" -} - -try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for VBR inventories ..." - $VbrVmInventories = Test-Path /mnt/wfs/inventory/VbrInv_*.csv -} catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] No VBR VM Inventories ?!" - Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" - exit -} - -if ($VbrVmInventories) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Import/Export VBR inventories ..." - try { - Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force #TODO sort LastRestorePoint - } catch { - AltAndCatchFire "Import/Export VBR inventories issue" - } -} - -Write-Host "$((Get-Date).ToString("o")) [INFO] End of VBR inventories processing ..." \ No newline at end of file From 3c875269198f3b167956bf18c128826cf49e6325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Mon, 4 Sep 2023 22:54:55 +0200 Subject: [PATCH 115/201] Update addVbrCrontab.sh --- var/www/scripts/addVbrCrontab.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/var/www/scripts/addVbrCrontab.sh b/var/www/scripts/addVbrCrontab.sh index 5535158e..73528525 100644 --- a/var/www/scripts/addVbrCrontab.sh +++ b/var/www/scripts/addVbrCrontab.sh @@ -2,5 +2,4 @@ sessionFile="/tmp/vbr_$(sed s/\\./_/g <<<$1).key" crontabFile="/etc/cron.d/vbr_$(sed s/\\./_/g <<<$1)" echo "*/5 * * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/veeam/VbrPullStatistics.ps1 -credstore /mnt/wfs/inventory/vbrpscredentials.xml -server $1 -sessionfile $sessionFile >/dev/null 2>&1" >> $crontabFile -echo "3 */1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/veeam/VbrVmInventory.ps1 >/dev/null 2>&1" > "/etc/cron.d/VbrVmInventory" service cron reload \ No newline at end of file From 10b934d2a84294d8c01f9263bf8b4c74132340e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Mon, 4 Sep 2023 22:55:02 +0200 Subject: [PATCH 116/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 5a2b12ac..8794a67c 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.30" +$ScriptVersion = "0.9.32" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -348,8 +348,9 @@ if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Building $($Server) VBR DataTable and inventory ..." foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { - $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name - $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ + if ($VbrSessions5[$VbrObjectRestorePoint.backupId].name) { + $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name + $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ $vm_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name if ($ViVmInventoryTable[$vm_name]) { $vcenter_name = NameCleaner $ViVmInventoryTable[$vm_name].vCenter @@ -375,6 +376,7 @@ if ($($VbrJobsStates.data)) { $VbrVmInventoryTable.add($vm_name,$VbrObjectInventoryInfo) } } + } } if ($VbrVmInventoryTable.Values) { @@ -385,6 +387,13 @@ if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [EROR] VbrVmInventory import issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } + Write-Host "$((Get-Date).ToString("o")) [INFO] Import/Export VBR inventories ..." + try { + Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force #TODO sort LastRestorePoint + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Import/Export VBR inventories issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } } else { Write-Host "$((Get-Date).ToString("o")) [EROR] No VbrVmInventoryTable values ?!" } From 735f41628a606e0ce7e41fd49910d7f113db142a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 6 Sep 2023 09:25:36 +0200 Subject: [PATCH 117/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 8794a67c..2424d762 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -432,7 +432,7 @@ if ($($VbrJobsStates.data)) { $VbrDataTable["veeam.vbr.$vbrserver_name.exec.duration"] = $($(Get-Date).ToUniversalTime() - $ExecStart).TotalSeconds Write-Host "$((Get-Date).ToString("o")) [INFO] Sending veeam data to Graphite for VBR server $Server ..." - # Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrDataTable -DateTime $ExecStart + Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrDataTable -DateTime $ExecStart Write-Host "$((Get-Date).ToString("o")) [INFO] End of VBR server $server processing ..." From 7ff44055ebc8a7d932b406db7d2cc137139e0531 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 7 Sep 2023 12:22:04 +0200 Subject: [PATCH 118/201] Update Veeam_Job_Result.json --- var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json index 1d981cee..2bd6dbf4 100644 --- a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json @@ -88,7 +88,7 @@ "hide": false, "refCount": 0, "refId": "B", - "target": "aliasByNode(removeEmptySeries(veeam.vbr.$vbrserver.job.$job.result), 4)" + "target": "sortByName(aliasByNode(removeEmptySeries(veeam.vbr.$vbrserver.job.$job.result), 4), false)" } ], "title": "Job Result", From 5176eb4b3b554a4aba56d2ac82688498329ae862 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 7 Sep 2023 12:22:06 +0200 Subject: [PATCH 119/201] Update Veeam_Job_Status.json --- var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json index 4663a16a..7e5a3283 100644 --- a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json @@ -87,7 +87,7 @@ { "datasource": "local", "refId": "A", - "target": "aliasByNode(removeEmptySeries(veeam.vbr.$vbrserver.job.$job.status), 4)", + "target": "sortByName(aliasByNode(removeEmptySeries(veeam.vbr.$vbrserver.job.$job.status), 4), false)", "textEditor": false } ], From 591da5d4be7caf7582a25f766498182c622e18c4 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 15 Sep 2023 14:45:26 +0200 Subject: [PATCH 120/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index d1726879..0523f770 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.80" +$ScriptVersion = "0.9.81" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -273,12 +273,12 @@ if ($ViServersList.count -gt 0) { $ViVmInfo.Cluster = $VmCluster $ViVmInfo.IP = $VmIpAddress -join " ; " $ViVmInfo.PortGroup = $VmNet -join " ; " - $ViVmInfo.Committed_GB = [math]::round($Vm.Summary.Storage.committed/1GB,1) - $ViVmInfo.Allocated_GB = [math]::round(($Vm.Summary.Storage.Committed + $Vm.Summary.Storage.Uncommitted)/1GB,1) + $ViVmInfo.Committed_GB = [math]::round($Vm.Summary.Storage.committed/1GB,0) + $ViVmInfo.Allocated_GB = [math]::round(($Vm.Summary.Storage.Committed + $Vm.Summary.Storage.Uncommitted)/1GB,0) $ViVmInfo.MAC = ($Vm.Config.Hardware.Device|?{$_.MacAddress}).MacAddress -join " ; " $ViVmInfo.GuestId = $VmGuestId $ViVmInfo.vCPU = $Vm.Config.Hardware.NumCPU - $ViVmInfo.vRAM_GB = [math]::round($Vm.Config.Hardware.MemoryMB/1KB,1) + $ViVmInfo.vRAM_GB = [math]::round($Vm.Config.Hardware.MemoryMB/1KB,0) $ViVmInfo.PowerState = $Vm.Runtime.PowerState $ViVmInfo.vmxPath = $Vm.summary.config.vmPathName $ViVmInfo.Folder = $VmPath @@ -324,7 +324,7 @@ if ($ViServersList.count -gt 0) { $ViEsxInfo.Model = $Esx.Summary.Hardware.Model $ViEsxInfo.SerialNumber = $EsxServiceTag $ViEsxInfo.State = $EsxState - $ViEsxInfo.RAM_GB = [math]::round($Esx.Summary.Hardware.MemorySize/1GB,1) + $ViEsxInfo.RAM_GB = [math]::round($Esx.Summary.Hardware.MemorySize/1GB,0) $ViEsxInfo.CPU = $Esx.Summary.Hardware.CpuModel $ViEsxInfo.Cores = $Esx.Summary.Hardware.NumCpuCores $ViEsxInfo.vmk0Ip = ($Esx.Config.Network.Vnic|?{$_.Device -eq "vmk0"}).Spec.Ip.IpAddress @@ -340,9 +340,9 @@ if ($ViServersList.count -gt 0) { $ViDatastoreInfo.vCenter = $ViServer $ViDatastoreInfo.Datastore = $($Datastore.name) $ViDatastoreInfo.Type = $($Datastore.Summary.Type) - $ViDatastoreInfo.Capacity_GB = $([math]::round($Datastore.Summary.Capacity/1GB,1)) - $ViDatastoreInfo.FreeSpace_GB = $([math]::round($Datastore.Summary.FreeSpace/1GB,1)) - $ViDatastoreInfo."Usage_%" = $([math]::round(($Datastore.Summary.Capacity - $Datastore.Summary.FreeSpace) * 100 / $Datastore.Summary.Capacity,1)) + $ViDatastoreInfo.Capacity_GB = $([math]::round($Datastore.Summary.Capacity/1GB,0)) + $ViDatastoreInfo.FreeSpace_GB = $([math]::round($Datastore.Summary.FreeSpace/1GB,0)) + $ViDatastoreInfo."Usage_%" = $([math]::round(($Datastore.Summary.Capacity - $Datastore.Summary.FreeSpace) * 100 / $Datastore.Summary.Capacity,0)) $ViDatastoreInfo.Url = $($Datastore.Summary.Url) $ViDatastoresInfos += $ViDatastoreInfo From 1f1210b537e481fc185a4bd324c9d8735e262f03 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 15 Sep 2023 17:06:56 +0200 Subject: [PATCH 121/201] Update VMware_All_Cluster_Top_N_VM_Stats.json --- .../dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json index ef78bd61..bbfe88d4 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json @@ -1087,7 +1087,7 @@ "error": false, "fill": 0, "grid": {}, - "id": 14, + "id": 15, "legend": { "avg": false, "current": false, From 86991afaed7834d02046edb2868026476c762ea4 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 15 Sep 2023 17:08:01 +0200 Subject: [PATCH 122/201] Update ViPullStatistics.ps1 #354 --- opt/sexigraf/ViPullStatistics.ps1 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 1dc0829b..89125251 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1020" +$ScriptVersion = "0.9.1021" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -463,7 +463,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { "storageAdapter.write.average", "power.power.average", "datastore.datastoreVMObservedLatency.latest", - # "datastore.totalWriteLatency.average", + "datastore.totalWriteLatency.average", # "datastore.totalReadLatency.average", "datastore.numberWriteAveraged.average", "datastore.numberReadAveraged.average", @@ -1162,6 +1162,10 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_datastore_latency_raw = $HostMultiStats[$PerfCounterTable["datastore.datastoreVMObservedLatency.latest"]][$vcenter_cluster_datastore_hosts.value]|%{$_[$vcenter_cluster_datastore_uuid]} $vcenter_cluster_datastore_latency = GetMedian $vcenter_cluster_datastore_latency_raw + if ($vcenter_cluster_datastore_latency -eq 0) { + $vcenter_cluster_datastore_latency_raw = $HostMultiStats[$PerfCounterTable["datastore.totalWriteLatency.average"]][$vcenter_cluster_datastore_hosts.value]|%{$_[$vcenter_cluster_datastore_uuid]} + $vcenter_cluster_datastore_latency = $(GetMedian $vcenter_cluster_datastore_latency_raw) * 1000 + } $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.iorm.sizeNormalizedDatastoreLatency", $vcenter_cluster_datastore_latency) $vcenter_cluster_datastores_latency += $vcenter_cluster_datastore_latency From 1579bffcaa60f76dc5ad2da731a2ec198c7f7afa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 16 Sep 2023 22:57:30 +0200 Subject: [PATCH 123/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 2424d762..7af2f70c 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.32" +$ScriptVersion = "0.9.34" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -164,7 +164,11 @@ if ($($VbrJobsStates.data)) { $vbrserver_name = NameCleaner $Server Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VBR Server $Server ..." + $VbrJobsStatesTable = @{} foreach ($VbrJobState in $VbrJobsStates.data) { + try { + $VbrJobsStatesTable.add($VbrJobState.id,$VbrJobState) + } catch {} $job_name = NameCleaner $VbrJobState.name if ($VbrJobState.status -eq "running") { $VbrJobStateStatus = 0 @@ -348,7 +352,7 @@ if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [INFO] Building $($Server) VBR DataTable and inventory ..." foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { - if ($VbrSessions5[$VbrObjectRestorePoint.backupId].name) { + if ($VbrJobsStatesTable[$VbrSessions5[$VbrObjectRestorePoint.backupId].jobId]) { $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ $vm_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name @@ -360,6 +364,7 @@ if ($($VbrJobsStates.data)) { $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].ESX } $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.objectRestorePoints"] ++ + $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.restorePointsCount"] ++ $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount $VbrObjectInventoryInfo = "" | Select-Object VbrServer, JobName, vCenter, Cluster, VM, RestorePointsCount, LastRestorePoint @@ -389,7 +394,7 @@ if ($($VbrJobsStates.data)) { } Write-Host "$((Get-Date).ToString("o")) [INFO] Import/Export VBR inventories ..." try { - Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force #TODO sort LastRestorePoint + Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Sort-Object LastRestorePoint -Descending | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] Import/Export VBR inventories issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" From f3283dcfc54be731a5c6d34c0bf45755c9c10567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 21 Sep 2023 01:20:13 +0200 Subject: [PATCH 124/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 7af2f70c..e4842198 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.34" +$ScriptVersion = "0.9.35" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -363,8 +363,7 @@ if ($($VbrJobsStates.data)) { } else { $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].ESX } - $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.objectRestorePoints"] ++ - $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.restorePointsCount"] ++ + $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.objectRestorePoints"] ++ $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount $VbrObjectInventoryInfo = "" | Select-Object VbrServer, JobName, vCenter, Cluster, VM, RestorePointsCount, LastRestorePoint From 1d667eff2968464fbd7a4f09644dccb309c35251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 21 Sep 2023 01:20:17 +0200 Subject: [PATCH 125/201] Update Veeam_Restore_Point.json --- var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json index ffc10f5c..abc56946 100644 --- a/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json @@ -134,7 +134,7 @@ { "datasource": "local", "refId": "A", - "target": "groupByNode(removeEmptySeries(veeam.vi.$vcenter.$cluster.vm.*.objectRestorePoints), 3, 'total')" + "target": "groupByNode(removeEmptySeries(veeam.vi.$vcenter.$cluster.objectRestorePoints), 3, 'total')" } ], "thresholds": [], From 88e94c47fa57e48add4119f4af3f732e27daba11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 22 Sep 2023 09:25:50 +0200 Subject: [PATCH 126/201] Update VMware_All_Standalone_ESX_VM_Stats.json --- .../VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json index c04f0bf3..2950da2c 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json @@ -652,7 +652,7 @@ "targets": [ { "refId": "A", - "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskiops, '$N')), 5)", + "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskIOPS, '$N')), 5)", "textEditor": true } ], From 806354b1e0ff12116e589b91606e3d463492e8de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 22 Sep 2023 09:25:53 +0200 Subject: [PATCH 127/201] Update VMware_All_Standalone_ESX_Top_N_VM_Stats.json --- .../VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json index eb80ef8a..971135b2 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json @@ -652,7 +652,7 @@ "targets": [ { "refId": "A", - "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskiops, '$N')), 5)", + "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskIOPS, '$N')), 5)", "textEditor": true } ], From 242231ef9e1e423e1c865a66a2d86aacaac1a499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 22 Sep 2023 09:25:55 +0200 Subject: [PATCH 128/201] Update VMware_All_Cluster_VM_Stats.json --- .../dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json index b3c7912f..be25962f 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_VM_Stats.json @@ -652,7 +652,7 @@ "targets": [ { "refId": "A", - "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.vm.$vm.fatstats.diskiops, 5)", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.vm.$vm.fatstats.diskIOPS, 5)", "textEditor": false } ], From 37a9106b63e3b7494f014dec2c8f9ad8e9370013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 22 Sep 2023 09:26:00 +0200 Subject: [PATCH 129/201] Update VMware_All_Cluster_Top_N_VM_Stats.json --- .../dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json index bbfe88d4..593b1b1c 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Cluster_Top_N_VM_Stats.json @@ -652,7 +652,7 @@ "targets": [ { "refId": "A", - "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$cluster.vm.$vm.fatstats.diskiops, '$N')), 5)", + "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$cluster.vm.$vm.fatstats.diskIOPS, '$N')), 5)", "textEditor": true } ], From 820ad60946166366ff743b2de9dbdffe7d72dc42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 27 Sep 2023 00:30:04 +0200 Subject: [PATCH 130/201] #355 --- ...e_Usage_Pct.json => VMware_Multi_Datastore_Usage_Gauge.json} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename var/lib/grafana/dashboards/VMware_Datastore/{VMware_Multi_Datastore_Usage_Pct.json => VMware_Multi_Datastore_Usage_Gauge.json} (99%) diff --git a/var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Pct.json b/var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Gauge.json similarity index 99% rename from var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Pct.json rename to var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Gauge.json index e407441c..635dc93e 100644 --- a/var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Pct.json +++ b/var/lib/grafana/dashboards/VMware_Datastore/VMware_Multi_Datastore_Usage_Gauge.json @@ -242,7 +242,7 @@ "type": "timepicker" }, "timezone": "browser", - "title": "VMware Multi Datastore Usage %", + "title": "VMware Multi Datastore Gauge", "uid": "6e3fb4e8", "variables": { "list": [] From cf987e5e0a0dc6a91a219747f79060bd97da3d87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 27 Sep 2023 02:04:28 +0200 Subject: [PATCH 131/201] Update ViPullStatistics.ps1 #350 --- opt/sexigraf/ViPullStatistics.ps1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 89125251..236da278 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1021" +$ScriptVersion = "0.9.1023" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -462,7 +462,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { "storageAdapter.read.average", "storageAdapter.write.average", "power.power.average", - "datastore.datastoreVMObservedLatency.latest", + "datastore.sizeNormalizedDatastoreLatency.average", "datastore.totalWriteLatency.average", # "datastore.totalReadLatency.average", "datastore.numberWriteAveraged.average", @@ -1160,7 +1160,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $vcenter_cluster_datastore_uuid = $vcenter_cluster_datastore.summary.url.split("/")[-2] - $vcenter_cluster_datastore_latency_raw = $HostMultiStats[$PerfCounterTable["datastore.datastoreVMObservedLatency.latest"]][$vcenter_cluster_datastore_hosts.value]|%{$_[$vcenter_cluster_datastore_uuid]} + $vcenter_cluster_datastore_latency_raw = $HostMultiStats[$PerfCounterTable["datastore.sizeNormalizedDatastoreLatency.average"]][$vcenter_cluster_datastore_hosts.value]|%{$_[$vcenter_cluster_datastore_uuid]} $vcenter_cluster_datastore_latency = GetMedian $vcenter_cluster_datastore_latency_raw if ($vcenter_cluster_datastore_latency -eq 0) { $vcenter_cluster_datastore_latency_raw = $HostMultiStats[$PerfCounterTable["datastore.totalWriteLatency.average"]][$vcenter_cluster_datastore_hosts.value]|%{$_[$vcenter_cluster_datastore_uuid]} @@ -1660,7 +1660,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($vcenter_standalone_host_datastore.summary.type -notmatch "vsan") { $vcenter_standalone_host_datastore_uuid = $vcenter_standalone_host_datastore.summary.url.split("/")[-2] - $vcenter_standalone_host_datastore_latency = $HostMultiStats[$PerfCounterTable["datastore.datastoreVMObservedLatency.latest"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_datastore_uuid] + $vcenter_standalone_host_datastore_latency = $HostMultiStats[$PerfCounterTable["datastore.sizeNormalizedDatastoreLatency.latest"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_datastore_uuid] $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.datastore.$vcenter_standalone_host_datastore_name.iorm.sizeNormalizedDatastoreLatency", $vcenter_standalone_host_datastore_latency) $vcenter_standalone_host_datastore_iops_w = $HostMultiStats[$PerfCounterTable["datastore.numberWriteAveraged.average"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_datastore_uuid] @@ -2088,7 +2088,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { "storageAdapter.read.average", "storageAdapter.write.average" # "power.power.average", - # "datastore.datastoreVMObservedLatency.latest", + # "datastore.sizeNormalizedDatastoreLatency.latest", # "datastore.numberWriteAveraged.average", # "datastore.numberReadAveraged.average" ) From da39d00cd1a8ff17bf792482932d7a5d5daa2f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 27 Sep 2023 16:17:48 +0200 Subject: [PATCH 132/201] Update ViPullStatistics.ps1 #347 --- opt/sexigraf/ViPullStatistics.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 236da278..8e383984 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1023" +$ScriptVersion = "0.9.1024" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -1124,7 +1124,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_datastores_count ++ - $vcenter_cluster_datastore_hosts = ($vcenter_cluster_datastore.Host|?{$_.mountinfo.Accessible}).key + $vcenter_cluster_datastore_hosts = $vcenter_vmhosts_h[$(($vcenter_cluster_datastore.Host|?{$_.mountinfo.Accessible}).key).value].moref if ($vcenter_cluster_datastore.summary.uncommitted -ge 0) { $vcenter_cluster_datastore_uncommitted = $vcenter_cluster_datastore.summary.uncommitted @@ -1160,7 +1160,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $vcenter_cluster_datastore_uuid = $vcenter_cluster_datastore.summary.url.split("/")[-2] - $vcenter_cluster_datastore_latency_raw = $HostMultiStats[$PerfCounterTable["datastore.sizeNormalizedDatastoreLatency.average"]][$vcenter_cluster_datastore_hosts.value]|%{$_[$vcenter_cluster_datastore_uuid]} + $vcenter_cluster_datastore_latency_raw = $($HostMultiStats[$PerfCounterTable["datastore.sizeNormalizedDatastoreLatency.average"]][$vcenter_cluster_datastore_hosts.value])|?{$_.count -gt 0}|%{$_[$vcenter_cluster_datastore_uuid]} #347 $vcenter_cluster_datastore_latency = GetMedian $vcenter_cluster_datastore_latency_raw if ($vcenter_cluster_datastore_latency -eq 0) { $vcenter_cluster_datastore_latency_raw = $HostMultiStats[$PerfCounterTable["datastore.totalWriteLatency.average"]][$vcenter_cluster_datastore_hosts.value]|%{$_[$vcenter_cluster_datastore_uuid]} From 6474719c096a404c7b523c81afb22864477b5bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 27 Sep 2023 22:07:40 +0200 Subject: [PATCH 133/201] Update ViPullStatistics.ps1 #347 --- opt/sexigraf/ViPullStatistics.ps1 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 8e383984..528010c0 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1024" +$ScriptVersion = "0.9.1025" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -668,9 +668,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_hosts_hba_bytesWrite = 0 $vcenter_cluster_hosts_power_usage = 0 $vcenter_cluster_hosts_vms_dead = 0 + $vcenter_cluster_hosts_h = @{} foreach ($vcenter_cluster_host in $vcenter_vmhosts_h[$vcenter_cluster.Host.value]|?{$_}) { + $vcenter_cluster_hosts_h.add($vcenter_cluster_host.moref.value,$vcenter_cluster_host) + $vcenter_cluster_host_name = $vcenter_cluster_host.config.network.dnsConfig.hostName.ToLower() ### why not $vcenter_cluster_host.name.split(".")[0].ToLower() ? if ($vcenter_cluster_host_name -match "localhost") { $vcenter_cluster_host_name = NameCleaner $vcenter_cluster_host.name ### previously vmk0 ip cleaned @@ -1124,7 +1127,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_datastores_count ++ - $vcenter_cluster_datastore_hosts = $vcenter_vmhosts_h[$(($vcenter_cluster_datastore.Host|?{$_.mountinfo.Accessible}).key).value].moref + $vcenter_cluster_datastore_hosts = $vcenter_cluster_hosts_h[$(($vcenter_cluster_datastore.Host|?{$_.mountinfo.Accessible}).key).value].moref if ($vcenter_cluster_datastore.summary.uncommitted -ge 0) { $vcenter_cluster_datastore_uncommitted = $vcenter_cluster_datastore.summary.uncommitted From 73a7cb4004a18cdc1bff8092d03d370a54838406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 28 Sep 2023 21:57:01 +0200 Subject: [PATCH 134/201] Update ViPullStatistics.ps1 --- opt/sexigraf/ViPullStatistics.ps1 | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 528010c0..cc1516a1 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -227,7 +227,7 @@ if ($SessionFile) { } if (!$($global:DefaultVIServer)) { - ### XXX The session is not authenticated + ### The session is not authenticated # https://github.com/guyrleech/VMware/blob/master/VMware%20GUI.ps1#L2940 try { $createstorexml = New-Object -TypeName XML @@ -674,7 +674,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_hosts_h.add($vcenter_cluster_host.moref.value,$vcenter_cluster_host) - $vcenter_cluster_host_name = $vcenter_cluster_host.config.network.dnsConfig.hostName.ToLower() ### why not $vcenter_cluster_host.name.split(".")[0].ToLower() ? + $vcenter_cluster_host_name = $vcenter_cluster_host.config.network.dnsConfig.hostName.ToLower() ### to avoid esx registered by ip if ($vcenter_cluster_host_name -match "localhost") { $vcenter_cluster_host_name = NameCleaner $vcenter_cluster_host.name ### previously vmk0 ip cleaned @@ -687,7 +687,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($vcenter_cluster_host_real_vm_count -gt $vcenter_cluster_host_connected_vm_count) { $vcenter_cluster_hosts_vms_dead += $vcenter_cluster_host_real_vm_count - $vcenter_cluster_host_connected_vm_count } - ### XXX use $vcenter_resource_pools_owner_vms_h + ### TODO use $vcenter_resource_pools_owner_vms_h ? } if ($vcenter_cluster_host.config.product.version -and $vcenter_cluster_host.config.product.build -and $vcenter_cluster_host.summary.hardware.cpuModel -and $vcenter_cluster_host.summary.hardware.NumCpuPkgs) { @@ -733,7 +733,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_hosts_net_bytesTx += $vcenter_cluster_host_vmnic_bytesTx $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.net.$vcenter_cluster_host_vmnic_name.bytesRx", $vcenter_cluster_host_vmnic_bytesRx) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.net.$vcenter_cluster_host_vmnic_name.bytesTx", $vcenter_cluster_host_vmnic_bytesTx) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.net.$vcenter_cluster_host_vmnic_name.linkSpeed", $vcenter_cluster_host_vmnic.linkSpeed.speedMb) ### XXX Still usefull? + # $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.net.$vcenter_cluster_host_vmnic_name.linkSpeed", $vcenter_cluster_host_vmnic.linkSpeed.speedMb) } $vcenter_cluster_host_vmnic_droppedRx = $HostMultiStats[$PerfCounterTable["net.droppedRx.summation"]][$vcenter_cluster_host.moref.value][$vcenter_cluster_host_vmnic_name] @@ -765,7 +765,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } try { - foreach ($vcenter_cluster_host_vmhba in $vcenter_cluster_host.config.storageDevice.hostBusAdapter) { ### TODO filter vmhba based on Config.MultipathState.Path + foreach ($vcenter_cluster_host_vmhba in $vcenter_cluster_host.config.storageDevice.hostBusAdapter) { $vcenter_cluster_host_vmhba_name = $vcenter_cluster_host_vmhba.device $vcenter_cluster_host_vmhba_bytesRead = $HostMultiStats[$PerfCounterTable["storageAdapter.read.average"]][$vcenter_cluster_host.moref.value][$vcenter_cluster_host_vmhba_name] $vcenter_cluster_host_vmhba_bytesWrite = $HostMultiStats[$PerfCounterTable["storageAdapter.write.average"]][$vcenter_cluster_host.moref.value][$vcenter_cluster_host_vmhba_name] @@ -853,8 +853,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } try { - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedCpuFairness", $vcenter_cluster_host.summary.quickStats.distributedCpuFairness) ### ToDo - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedMemoryFairness", $vcenter_cluster_host.summary.quickStats.distributedMemoryFairness) ### ToDo + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedCpuFairness", $vcenter_cluster_host.summary.quickStats.distributedCpuFairness) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedMemoryFairness", $vcenter_cluster_host.summary.quickStats.distributedMemoryFairness) } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name Distributed Fairness quickstats issue in cluster $vcenter_cluster_name" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -956,7 +956,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } foreach ($vcenter_cluster_vm_file in $vcenter_cluster_vm_files) { - if(!$vcenter_cluster_vms_files_dedup[$vcenter_cluster_vm_file.name]) { ### XXX would need name & moref + if(!$vcenter_cluster_vms_files_dedup[$vcenter_cluster_vm_file.name]) { ### TODO would need name & moref $vcenter_cluster_vms_files_dedup[$vcenter_cluster_vm_file.name] = $vcenter_cluster_vm_file.size if ($vcenter_cluster_vm_has_snap -and (($vcenter_cluster_vm_file.name -match '-[0-9]{6}-delta\.vmdk') -or ($vcenter_cluster_vm_file.name -match '-[0-9]{6}-sesparse\.vmdk'))) { $vcenter_cluster_vms_files_dedup_total["snapshotExtent"] += $vcenter_cluster_vm_file.size @@ -1148,8 +1148,6 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.summary.vmdkCount", $vcenter_cluster_vmdk_per_ds[$vcenter_cluster_datastore_name]) } - ### XXX if vmdkCount -gt 0 - $vcenter_cluster_datastores_capacity += $vcenter_cluster_datastore.summary.capacity $vcenter_cluster_datastores_freeSpace += $vcenter_cluster_datastore.summary.freeSpace $vcenter_cluster_datastores_uncommitted += $vcenter_cluster_datastore_uncommitted @@ -1426,8 +1424,6 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Write-Host "$((Get-Date).ToString("o")) [WARN] Empty VsanPerfQuery in cluster $vcenter_cluster_name" } - ### XXX add check against multiple vsan datastore on the same cluster - try { Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing SmartStatsSummary in cluster $vcenter_cluster_name (v6.7+) ..." # https://www.virtuallyghetto.com/2017/04/getting-started-wthe-new-powercli-6-5-1-get-vsanview-cmdlet.html @@ -1516,7 +1512,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VsanObjectIdentityAndHealth in cluster $vcenter_cluster_name ..." - $vcenter_cluster_ObjectIdentities = $VsanObjectSystem.VsanQueryObjectIdentities($vcenter_cluster.moref,$null,$null,$true,$false,$false) ### XXX #ToDo optimize + $vcenter_cluster_ObjectIdentities = $VsanObjectSystem.VsanQueryObjectIdentities($vcenter_cluster.moref,$null,$null,$true,$false,$false) ### TODO optimize if ($vcenter_cluster_ObjectIdentities.Health.ObjectHealthDetail) { foreach ($ObjectHealth in $vcenter_cluster_ObjectIdentities.Health.ObjectHealthDetail) { if ($ObjectHealth.NumObjects -gt 0) { @@ -1698,7 +1694,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } try { - foreach ($vcenter_standalone_host_vmhba in $vcenter_standalone_host.config.storageDevice.hostBusAdapter) { ### TODO filter vmhba based on Config.MultipathState.Path + foreach ($vcenter_standalone_host_vmhba in $vcenter_standalone_host.config.storageDevice.hostBusAdapter) { $vcenter_standalone_host_vmhba_name = $vcenter_standalone_host_vmhba.device $vcenter_standalone_host_vmhba_bytesRead = $HostMultiStats[$PerfCounterTable["storageAdapter.read.average"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_vmhba_name] $vcenter_standalone_host_vmhba_bytesWrite = $HostMultiStats[$PerfCounterTable["storageAdapter.write.average"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_vmhba_name] @@ -1794,7 +1790,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } foreach ($vcenter_standalone_host_vm_file in $vcenter_standalone_host_vm_files) { - if(!$vcenter_standalone_host_vms_files_dedup[$vcenter_standalone_host_vm_file.name]) { ### XXX would need name & moref + if(!$vcenter_standalone_host_vms_files_dedup[$vcenter_standalone_host_vm_file.name]) { ### TODO would need name & moref $vcenter_standalone_host_vms_files_dedup[$vcenter_standalone_host_vm_file.name] = $vcenter_standalone_host_vm_file.size if ($vcenter_standalone_host_vm_has_snap -and (($vcenter_standalone_host_vm_file.name -match '-[0-9]{6}-delta\.vmdk') -or ($vcenter_standalone_host_vm_file.name -match '-[0-9]{6}-sesparse\.vmdk'))) { $vcenter_standalone_host_vms_files_dedup_total["snapshotExtent"] += $vcenter_standalone_host_vm_file.size @@ -2252,7 +2248,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } try { - foreach ($unmanaged_host_vmhba in $unmanaged_host.config.storageDevice.hostBusAdapter) { ### XXX dead paths from config.storageDevice.HostBusAdapter to add + foreach ($unmanaged_host_vmhba in $unmanaged_host.config.storageDevice.hostBusAdapter) { $unmanaged_host_vmhba_name = $unmanaged_host_vmhba.device $unmanaged_host_vmhba_bytesRead = $HostMultiStats[$PerfCounterTable["storageAdapter.read.average"]][$unmanaged_host.moref.value][$unmanaged_host_vmhba_name] $unmanaged_host_vmhba_bytesWrite = $HostMultiStats[$PerfCounterTable["storageAdapter.write.average"]][$unmanaged_host.moref.value][$unmanaged_host_vmhba_name] @@ -2346,7 +2342,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } foreach ($unmanaged_host_vm_file in $unmanaged_host_vm_files) { - if(!$unmanaged_host_vms_files_dedup[$unmanaged_host_vm_file.name]) { ### XXX would need name & moref + if(!$unmanaged_host_vms_files_dedup[$unmanaged_host_vm_file.name]) { ### TODO would need name & moref $unmanaged_host_vms_files_dedup[$unmanaged_host_vm_file.name] = $unmanaged_host_vm_file.size if ($unmanaged_host_vm_has_snap -and (($unmanaged_host_vm_file.name -match '-[0-9]{6}-delta\.vmdk') -or ($unmanaged_host_vm_file.name -match '-[0-9]{6}-sesparse\.vmdk'))) { $unmanaged_host_vms_files_dedup_total["snapshotExtent"] += $unmanaged_host_vm_file.size From 31b1b51afc6e522ac3baf1bcc77f5d65066e8783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 29 Sep 2023 15:02:23 +0200 Subject: [PATCH 135/201] Update Veeam_Job_Result.json --- .../dashboards/Veeam/Veeam_Job_Result.json | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json index 2bd6dbf4..f610d490 100644 --- a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json @@ -134,40 +134,6 @@ "sort": 0, "type": "query", "datasource": "local" - }, - { - "allValue": "*", - "current": {}, - "definition": "veeam.vi.*", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "vcenter", - "options": [], - "query": "veeam.vi.*", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query", - "datasource": "local" - }, - { - "allValue": "*", - "current": {}, - "definition": "veeam.vi.$vcenter.*", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "cluster", - "options": [], - "query": "veeam.vi.$vcenter.*", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query", - "datasource": "local" } ] }, From 28579e34630e6f0256ca8760fbc43b45eb767edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sun, 1 Oct 2023 13:18:07 +0200 Subject: [PATCH 136/201] Create VMware_Multi_Cluster_DRS_Score.json #228 --- .../VMware_Multi_Cluster_DRS_Score.json | 469 ++++++++++++++++++ 1 file changed, 469 insertions(+) create mode 100644 var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_DRS_Score.json diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_DRS_Score.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_DRS_Score.json new file mode 100644 index 00000000..3898b580 --- /dev/null +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_DRS_Score.json @@ -0,0 +1,469 @@ +{ + "__inputs": [ + { + "name": "DS_LOCAL", + "label": "local", + "description": "", + "type": "datasource", + "pluginId": "graphite", + "pluginName": "Graphite" + } + ], + "__elements": [], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "8.4.4" + }, + { + "type": "datasource", + "id": "graphite", + "name": "Graphite", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "iteration": 1696154080221, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "graphite", + "uid": "${DS_LOCAL}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "title": "$cluster", + "type": "row" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "VM", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "0_20" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "21_40" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "41_60" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "61_80" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "81_100" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ClusterDrsScore" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + }, + { + "id": "max", + "value": 100 + }, + { + "id": "min", + "value": 0 + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 2 + }, + { + "id": "custom.axisLabel", + "value": "" + }, + { + "id": "unit", + "value": "percent" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 19, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.4.4", + "repeat": "cluster", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "${DS_LOCAL}" + }, + "hide": false, + "refCount": 0, + "refId": "A", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.0_20, 6)", + "textEditor": false + }, + { + "datasource": { + "type": "graphite", + "uid": "${DS_LOCAL}" + }, + "hide": false, + "refCount": 0, + "refId": "B", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.21_40, 6)", + "textEditor": false + }, + { + "datasource": { + "type": "graphite", + "uid": "${DS_LOCAL}" + }, + "hide": false, + "refCount": 0, + "refId": "C", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.41_60, 6)", + "textEditor": false + }, + { + "datasource": { + "type": "graphite", + "uid": "${DS_LOCAL}" + }, + "hide": false, + "refCount": 0, + "refId": "D", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.61_80, 6)", + "textEditor": false + }, + { + "datasource": { + "type": "graphite", + "uid": "${DS_LOCAL}" + }, + "hide": false, + "refCount": 0, + "refId": "E", + "target": "aliasByNode(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.81_100, 6)", + "textEditor": false + }, + { + "datasource": { + "type": "graphite", + "uid": "${DS_LOCAL}" + }, + "hide": false, + "refCount": 0, + "refId": "F", + "target": "alias(vmw.$vcenter.$datacentre.$cluster.quickstats.drs.DrsScore, 'ClusterDrsScore')", + "textEditor": false + } + ], + "title": "DrsScoreBucket $cluster", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 35, + "style": "light", + "tags": [], + "templating": { + "list": [ + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": { + "type": "graphite", + "uid": "${DS_LOCAL}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "multiFormat": "glob", + "name": "vcenter", + "options": [], + "query": "vmw.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": { + "type": "graphite", + "uid": "${DS_LOCAL}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "multiFormat": "glob", + "name": "datacentre", + "options": [], + "query": "vmw.$vcenter.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "glob", + "allValue": "*", + "current": {}, + "datasource": { + "type": "graphite", + "uid": "${DS_LOCAL}" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "multi": true, + "multiFormat": "glob", + "name": "cluster", + "options": [], + "query": "vmw.$vcenter.$datacentre.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5m", + "15m", + "1h", + "2h", + "3h", + "6h", + "12h", + "24h" + ], + "status": "Stable", + "time_options": [ + "5m", + "15m", + "30m", + "1h", + "2h", + "1d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "VMware Cluster Multi DRS Score", + "uid": "whn1iUZIk", + "version": 11, + "weekStart": "" + } \ No newline at end of file From e9f7365046c953d7e33f6b94b21f7bec367224d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sun, 1 Oct 2023 13:18:10 +0200 Subject: [PATCH 137/201] Update Veeam_Restore_Point.json --- var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json index abc56946..1bce0575 100644 --- a/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json @@ -60,7 +60,7 @@ ], "thresholds": [], "timeRegions": [], - "title": "Restore Points Per Jobs", + "title": "Restore Points Per Job", "tooltip": { "shared": true, "sort": 2, From 75d55bd5e4217e4e7e93afcaf6ea5ae5cf6fea18 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 1 Oct 2023 13:45:42 +0200 Subject: [PATCH 138/201] timepicker alignement --- .../grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json | 2 +- var/lib/grafana/dashboards/SexiGraf/Netdata.json | 2 +- var/lib/grafana/dashboards/SexiGraf/SexiGraf_Web_Admin.json | 2 +- .../VMware_Cluster/VMware_All_Cluster_Capacity_Planning.json | 2 +- .../dashboards/VMware_Cluster/VMware_Multi_Cluster_vMotion.json | 2 +- .../dashboards/VMware_Inventory/VMware_Datastore_Inventory.json | 2 +- .../dashboards/VMware_Inventory/VMware_ESX_Inventory.json | 2 +- .../dashboards/VMware_Inventory/VMware_VBR_Inventory.json | 2 +- .../dashboards/VMware_Inventory/VMware_VM_Inventory.json | 2 +- var/lib/grafana/dashboards/VMware_vCenter/VMware_BroStats.json | 2 +- .../dashboards/VMware_vSAN/VMware_VSAN_Space_Usage_Report.json | 2 +- var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json | 2 +- var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json | 2 +- .../grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json | 2 +- var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json | 2 +- var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json b/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json index b9edf10b..0244ae31 100644 --- a/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json +++ b/var/lib/grafana/dashboards/SexiGraf/Graphite_Carbon_Metrics.json @@ -1423,7 +1423,7 @@ "list": [] }, "time": { - "from": "now-3h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/SexiGraf/Netdata.json b/var/lib/grafana/dashboards/SexiGraf/Netdata.json index a9d3788a..9fdd013d 100644 --- a/var/lib/grafana/dashboards/SexiGraf/Netdata.json +++ b/var/lib/grafana/dashboards/SexiGraf/Netdata.json @@ -67,7 +67,7 @@ } ], "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "templating": { diff --git a/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Web_Admin.json b/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Web_Admin.json index f8b42f57..939ef56f 100644 --- a/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Web_Admin.json +++ b/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Web_Admin.json @@ -33,7 +33,7 @@ } ], "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_Capacity_Planning.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_Capacity_Planning.json index 59c0a43e..afa1b6b9 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_Capacity_Planning.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_All_Cluster_Capacity_Planning.json @@ -210,7 +210,7 @@ } ], "time": { - "from": "now-24h", + "from": "now-1d", "to": "now" }, "templating": { diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_vMotion.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_vMotion.json index 348e9157..b8d42d8f 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_vMotion.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_vMotion.json @@ -152,7 +152,7 @@ } ], "time": { - "from": "now-1d", + "from": "now-1h", "to": "now" }, "templating": { diff --git a/var/lib/grafana/dashboards/VMware_Inventory/VMware_Datastore_Inventory.json b/var/lib/grafana/dashboards/VMware_Inventory/VMware_Datastore_Inventory.json index 33081726..13c031cd 100644 --- a/var/lib/grafana/dashboards/VMware_Inventory/VMware_Datastore_Inventory.json +++ b/var/lib/grafana/dashboards/VMware_Inventory/VMware_Datastore_Inventory.json @@ -77,7 +77,7 @@ "list": [] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_Inventory/VMware_ESX_Inventory.json b/var/lib/grafana/dashboards/VMware_Inventory/VMware_ESX_Inventory.json index 8cca5577..fc727e01 100644 --- a/var/lib/grafana/dashboards/VMware_Inventory/VMware_ESX_Inventory.json +++ b/var/lib/grafana/dashboards/VMware_Inventory/VMware_ESX_Inventory.json @@ -77,7 +77,7 @@ "list": [] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json index 872d312d..222c2c37 100644 --- a/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json +++ b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json @@ -77,7 +77,7 @@ "list": [] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_Inventory/VMware_VM_Inventory.json b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VM_Inventory.json index 9e167f33..5cc5456f 100644 --- a/var/lib/grafana/dashboards/VMware_Inventory/VMware_VM_Inventory.json +++ b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VM_Inventory.json @@ -77,7 +77,7 @@ "list": [] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_vCenter/VMware_BroStats.json b/var/lib/grafana/dashboards/VMware_vCenter/VMware_BroStats.json index 72a2af92..021a03f5 100644 --- a/var/lib/grafana/dashboards/VMware_vCenter/VMware_BroStats.json +++ b/var/lib/grafana/dashboards/VMware_vCenter/VMware_BroStats.json @@ -856,7 +856,7 @@ "list": [] }, "time": { - "from": "now-6h", + "from": "now-1h", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Space_Usage_Report.json b/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Space_Usage_Report.json index 31f53a19..0ac00ad1 100644 --- a/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Space_Usage_Report.json +++ b/var/lib/grafana/dashboards/VMware_vSAN/VMware_VSAN_Space_Usage_Report.json @@ -332,7 +332,7 @@ ] }, "time": { - "from": "now-3h", + "from": "now-1d", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json index f610d490..ba3d42bd 100644 --- a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Result.json @@ -138,7 +138,7 @@ ] }, "time": { - "from": "now-7d", + "from": "now-1d", "to": "now" }, "timepicker": {}, diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json index 7e5a3283..d0fe39bc 100644 --- a/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Job_Status.json @@ -139,7 +139,7 @@ ] }, "time": { - "from": "now-12h", + "from": "now-1d", "to": "now" }, "timepicker": {}, diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json index 38b23a69..312087a3 100644 --- a/var/lib/grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_Repository_Usage.json @@ -188,7 +188,7 @@ ] }, "time": { - "from": "now-24h", + "from": "now-1d", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json index 5c6f4026..58330fac 100644 --- a/var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Multi_SOBR_Usage.json @@ -188,7 +188,7 @@ ] }, "time": { - "from": "now-24h", + "from": "now-1d", "to": "now" }, "timepicker": { diff --git a/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json index 1bce0575..d62a20f2 100644 --- a/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json +++ b/var/lib/grafana/dashboards/Veeam/Veeam_Restore_Point.json @@ -247,7 +247,7 @@ ] }, "time": { - "from": "now-2d", + "from": "now-1d", "to": "now" }, "timepicker": {}, From a671180215334601c4c55782537dfc14ddad2274 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 1 Oct 2023 13:55:57 +0200 Subject: [PATCH 139/201] Update VMware_Multi_Cluster_DRS_Score.json #228 --- .../VMware_Multi_Cluster_DRS_Score.json | 104 ++---------------- 1 file changed, 12 insertions(+), 92 deletions(-) diff --git a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_DRS_Score.json b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_DRS_Score.json index 3898b580..b40f656c 100644 --- a/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_DRS_Score.json +++ b/var/lib/grafana/dashboards/VMware_Cluster/VMware_Multi_Cluster_DRS_Score.json @@ -1,55 +1,5 @@ { - "__inputs": [ - { - "name": "DS_LOCAL", - "label": "local", - "description": "", - "type": "datasource", - "pluginId": "graphite", - "pluginName": "Graphite" - } - ], - "__elements": [], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "8.4.4" - }, - { - "type": "datasource", - "id": "graphite", - "name": "Graphite", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, @@ -59,10 +9,7 @@ "panels": [ { "collapsed": false, - "datasource": { - "type": "graphite", - "uid": "${DS_LOCAL}" - }, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -273,10 +220,7 @@ "repeatDirection": "h", "targets": [ { - "datasource": { - "type": "graphite", - "uid": "${DS_LOCAL}" - }, + "datasource": null, "hide": false, "refCount": 0, "refId": "A", @@ -284,10 +228,7 @@ "textEditor": false }, { - "datasource": { - "type": "graphite", - "uid": "${DS_LOCAL}" - }, + "datasource": null, "hide": false, "refCount": 0, "refId": "B", @@ -295,10 +236,7 @@ "textEditor": false }, { - "datasource": { - "type": "graphite", - "uid": "${DS_LOCAL}" - }, + "datasource": null, "hide": false, "refCount": 0, "refId": "C", @@ -306,10 +244,7 @@ "textEditor": false }, { - "datasource": { - "type": "graphite", - "uid": "${DS_LOCAL}" - }, + "datasource": null, "hide": false, "refCount": 0, "refId": "D", @@ -317,10 +252,7 @@ "textEditor": false }, { - "datasource": { - "type": "graphite", - "uid": "${DS_LOCAL}" - }, + "datasource": null, "hide": false, "refCount": 0, "refId": "E", @@ -328,10 +260,7 @@ "textEditor": false }, { - "datasource": { - "type": "graphite", - "uid": "${DS_LOCAL}" - }, + "datasource": null, "hide": false, "refCount": 0, "refId": "F", @@ -353,10 +282,7 @@ "allFormat": "wildcard", "allValue": "*", "current": {}, - "datasource": { - "type": "graphite", - "uid": "${DS_LOCAL}" - }, + "datasource": null, "definition": "", "hide": 0, "includeAll": true, @@ -379,10 +305,7 @@ "allFormat": "wildcard", "allValue": "*", "current": {}, - "datasource": { - "type": "graphite", - "uid": "${DS_LOCAL}" - }, + "datasource": null, "definition": "", "hide": 0, "includeAll": true, @@ -405,10 +328,7 @@ "allFormat": "glob", "allValue": "*", "current": {}, - "datasource": { - "type": "graphite", - "uid": "${DS_LOCAL}" - }, + "datasource": null, "definition": "", "hide": 0, "includeAll": true, @@ -430,7 +350,7 @@ ] }, "time": { - "from": "now-2d", + "from": "now-1h", "to": "now" }, "timepicker": { From af2f214c6dfaa99564e5b88d37b9fbec23cf8c7b Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 1 Oct 2023 14:08:56 +0200 Subject: [PATCH 140/201] Update VsanPullStatistics.ps1 #143 --- opt/sexigraf/VsanPullStatistics.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opt/sexigraf/VsanPullStatistics.ps1 b/opt/sexigraf/VsanPullStatistics.ps1 index d877e91a..5d8e730e 100644 --- a/opt/sexigraf/VsanPullStatistics.ps1 +++ b/opt/sexigraf/VsanPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.79" +$ScriptVersion = "0.9.80" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -98,6 +98,8 @@ try { exit } +AltAndCatchFire "Deprecated script, now integrated to ViPullStatistics" + try { Write-Host "$((Get-Date).ToString("o")) [INFO] Importing PowerCli and Graphite PowerShell modules ..." Import-Module VMware.VimAutomation.Common, VMware.VimAutomation.Core, VMware.VimAutomation.Sdk, VMware.VimAutomation.Storage From f8c95787e2dbcc14586af664a68aca8ec58a6a74 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 1 Oct 2023 14:17:07 +0200 Subject: [PATCH 141/201] Update ViPullStatistics.ps1 #357 --- opt/sexigraf/ViPullStatistics.ps1 | 33 +++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index cc1516a1..e943392d 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1025" +$ScriptVersion = "0.9.1026" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -644,11 +644,32 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket -and $vcenter_cluster.summary.DrsScore) { # https://vdc-download.vmware.com/vmwb-repository/dcr-public/bf660c0a-f060-46e8-a94d-4b5e6ffc77ad/208bc706-e281-49b6-a0ce-b402ec19ef82/SDK/vsphere-ws/docs/ReferenceGuide/vim.ClusterComputeResource.Summary.html#numVmsPerDrsScoreBucket - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.0_20", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[0]) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.21_40", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[1]) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.41_60", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[2]) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.61_80", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[3]) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.81_100", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[4]) + if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket[0]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.0_20", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[0]) + } else { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.0_20", 0) + } + if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket[1]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.21_40", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[1]) + } else { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.21_40", 0) + } + if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket[2]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.41_60", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[2]) + } else { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.41_60", 0) + } + if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket[3]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.61_80", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[3]) + } else { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.61_80", 0) + } + if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket[4]) { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.81_100", $vcenter_cluster.summary.NumVmsPerDrsScoreBucket[4]) + } else { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.81_100", 0) + } + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.DrsScore", $vcenter_cluster.summary.DrsScore) } From 8caef2879244214412dca9ab0a1621b83036125c Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 1 Oct 2023 15:05:38 +0200 Subject: [PATCH 142/201] Update ViPullStatistics.ps1 #357 --- opt/sexigraf/ViPullStatistics.ps1 | 86 ++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 31 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index e943392d..5fa6a180 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1026" +$ScriptVersion = "0.9.1028" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -605,25 +605,55 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h = @{} - try { - $vcenter_cluster_pool_quickstats = $vcenter_resource_pools_h[$vcenter_cluster.moref.value].summary.quickStats - - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.ballooned", $vcenter_cluster_pool_quickstats.balloonedMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.compressed", $vcenter_cluster_pool_quickstats.compressedMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.consumedOverhead", $vcenter_cluster_pool_quickstats.consumedOverheadMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.guest", $vcenter_cluster_pool_quickstats.guestMemoryUsage) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.usage", $vcenter_cluster_pool_quickstats.hostMemoryUsage) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.demand", $vcenter_cluster_pool_quickstats.overallCpuDemand) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.usage", $vcenter_cluster_pool_quickstats.overallCpuUsage) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.private", $vcenter_cluster_pool_quickstats.privateMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.shared", $vcenter_cluster_pool_quickstats.sharedMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.swapped", $vcenter_cluster_pool_quickstats.swappedMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.effective", $vcenter_cluster.summary.effectiveMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.total", $vcenter_cluster.summary.totalMemory) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.effective", $vcenter_cluster.summary.effectiveCpu) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.total", $vcenter_cluster.summary.totalCpu) - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numVmotions", $vcenter_cluster.summary.numVmotions) + if ($vcenter_resource_pools_h[$vcenter_cluster.moref.value]) { + try { + $vcenter_cluster_pool_quickstats = $vcenter_resource_pools_h[$vcenter_cluster.moref.value].summary.quickStats + + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.ballooned", $vcenter_cluster_pool_quickstats.balloonedMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.compressed", $vcenter_cluster_pool_quickstats.compressedMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.consumedOverhead", $vcenter_cluster_pool_quickstats.consumedOverheadMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.guest", $vcenter_cluster_pool_quickstats.guestMemoryUsage) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.usage", $vcenter_cluster_pool_quickstats.hostMemoryUsage) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.demand", $vcenter_cluster_pool_quickstats.overallCpuDemand) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.usage", $vcenter_cluster_pool_quickstats.overallCpuUsage) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.private", $vcenter_cluster_pool_quickstats.privateMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.shared", $vcenter_cluster_pool_quickstats.sharedMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.swapped", $vcenter_cluster_pool_quickstats.swappedMemory) + + if ($vcenter_cluster_pool_quickstats.overallCpuUsage -gt 0 -and $vcenter_cluster.summary.effectiveCpu -gt 0) { + $vcenter_cluster_pool_quickstats_cpu = $vcenter_cluster_pool_quickstats.overallCpuUsage * 100 / $vcenter_cluster.summary.effectiveCpu + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.cpu.utilization", $vcenter_cluster_pool_quickstats_cpu) + } + + if ($vcenter_cluster_pool_quickstats.hostMemoryUsage -gt 0 -and $vcenter_cluster.summary.effectiveMemory -gt 0) { + $vcenter_cluster_pool_quickstats_ram = $vcenter_cluster_pool_quickstats.hostMemoryUsage * 100 / $vcenter_cluster.summary.effectiveMemory + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.mem.utilization", $vcenter_cluster_pool_quickstats_ram) + } + + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name quickstats collect issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + } else { + Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name root resource pool not found ?!" + } + + if ($vcenter_cluster.summary) { + try { + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.effective", $vcenter_cluster.summary.effectiveMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.mem.total", $vcenter_cluster.summary.totalMemory) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.effective", $vcenter_cluster.summary.effectiveCpu) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.total", $vcenter_cluster.summary.totalCpu) + $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numVmotions", $vcenter_cluster.summary.numVmotions) + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name summary collect issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } + } else { + Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name summary missing " + } + try { if($ClusterMultiStats[$PerfCounterTable["vmop.numSVMotion.latest"]][$vcenter_cluster.moref.value][""]) { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numSVMotions", $ClusterMultiStats[$PerfCounterTable["vmop.numSVMotion.latest"]][$vcenter_cluster.moref.value][""]) } @@ -631,17 +661,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if($ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value][""]) { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numXVMotions", $ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value][""]) } + } catch { + Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name xyzMotion collect issue" + Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + } - if ($vcenter_cluster_pool_quickstats.overallCpuUsage -gt 0 -and $vcenter_cluster.summary.effectiveCpu -gt 0) { - $vcenter_cluster_pool_quickstats_cpu = $vcenter_cluster_pool_quickstats.overallCpuUsage * 100 / $vcenter_cluster.summary.effectiveCpu - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.cpu.utilization", $vcenter_cluster_pool_quickstats_cpu) - } - - if ($vcenter_cluster_pool_quickstats.hostMemoryUsage -gt 0 -and $vcenter_cluster.summary.effectiveMemory -gt 0) { - $vcenter_cluster_pool_quickstats_ram = $vcenter_cluster_pool_quickstats.hostMemoryUsage * 100 / $vcenter_cluster.summary.effectiveMemory - $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.mem.utilization", $vcenter_cluster_pool_quickstats_ram) - } - + try { if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket -and $vcenter_cluster.summary.DrsScore) { # https://vdc-download.vmware.com/vmwb-repository/dcr-public/bf660c0a-f060-46e8-a94d-4b5e6ffc77ad/208bc706-e281-49b6-a0ce-b402ec19ef82/SDK/vsphere-ws/docs/ReferenceGuide/vim.ClusterComputeResource.Summary.html#numVmsPerDrsScoreBucket if ($vcenter_cluster.summary.NumVmsPerDrsScoreBucket[0]) { @@ -672,9 +697,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.DrsScore", $vcenter_cluster.summary.DrsScore) } - } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name root resource pool not found ?!" + Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name DrsScore collect issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } From ba1f5771e104449bc4cd85ff101b6160d0c317d6 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sun, 1 Oct 2023 15:42:55 +0200 Subject: [PATCH 143/201] Update ViPullStatistics.ps1 #357 --- opt/sexigraf/ViPullStatistics.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 5fa6a180..2dddaa5d 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1028" +$ScriptVersion = "0.9.1029" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -654,11 +654,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } try { - if($ClusterMultiStats[$PerfCounterTable["vmop.numSVMotion.latest"]][$vcenter_cluster.moref.value][""]) { + if($ClusterMultiStats[$PerfCounterTable["vmop.numSVMotion.latest"]][$vcenter_cluster.moref.value] -and $ClusterMultiStats[$PerfCounterTable["vmop.numSVMotion.latest"]][$vcenter_cluster.moref.value][""]) { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numSVMotions", $ClusterMultiStats[$PerfCounterTable["vmop.numSVMotion.latest"]][$vcenter_cluster.moref.value][""]) } - if($ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value][""]) { + if($ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value] -and $ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value][""]) { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numXVMotions", $ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value][""]) } } catch { From fe027aa1987acf9a1c0ddd28f253b8053baab214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 4 Oct 2023 23:44:17 +0200 Subject: [PATCH 144/201] Update graphite.conf #275 --- etc/apache2/sites-available/graphite.conf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/etc/apache2/sites-available/graphite.conf b/etc/apache2/sites-available/graphite.conf index 50374cbf..16e32e4e 100644 --- a/etc/apache2/sites-available/graphite.conf +++ b/etc/apache2/sites-available/graphite.conf @@ -118,8 +118,11 @@ ExtendedStatus On ProxyPass /sexiadmin/ ! Alias /sexiadmin/ /var/www/admin/ - Order allow,deny - Allow from all + RewriteEngine On + RewriteCond %{HTTP_REFERER} /sexigraf-web-admin [NC,OR] + RewriteCond %{HTTP_REFERER} /sexiadmin/ [NC] + RewriteRule .* - [L] + RewriteRule .* - [F] ProxyPass /favicon.ico ! From ad5c106618daff3ea31617749184adcccc5faa8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 5 Oct 2023 00:38:12 +0200 Subject: [PATCH 145/201] Update ViOfflineInventory.ps1 #275 --- opt/sexigraf/ViOfflineInventory.ps1 | 44 ++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 0523f770..bbb161fe 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.81" +$ScriptVersion = "0.9.83" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -361,6 +361,19 @@ if ($ViServersList.count -gt 0) { } if ($ViVmsInfos) { + + if ($(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv")) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViVmInventory.*.csv files ..." + $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} + If ($ExtraCsvFiles) { + try { + Get-ChildItem $ExtraCsvFiles.name | Remove-Item -Force -Confirm:$false + } catch { + AltAndCatchFire "Cannot remove extra csv files" + } + } + } + try { Write-Host "$((Get-Date).ToString("o")) [INFO] Building Vm Inventory CSV ..." $ViVmsInfosCsv = $ViVmsInfos|ConvertTo-Csv -NoTypeInformation -ErrorAction Stop @@ -385,6 +398,7 @@ if ($ViServersList.count -gt 0) { # } Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory file ..." $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.csv -Force -ErrorAction Stop + $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] VM Inventory issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -392,6 +406,19 @@ if ($ViServersList.count -gt 0) { } if ($ViEsxsInfos) { + + if ($(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv")) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViEsxInventory.*.csv files ..." + $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} + If ($ExtraCsvFiles) { + try { + Get-ChildItem $ExtraCsvFiles.name | Remove-Item -Force -Confirm:$false + } catch { + AltAndCatchFire "Cannot remove extra csv files" + } + } + } + try { Write-Host "$((Get-Date).ToString("o")) [INFO] Building ESX Inventory CSV ..." $ViEsxsInfosCsv = $ViEsxsInfos|ConvertTo-Csv -NoTypeInformation -ErrorAction Stop @@ -416,6 +443,7 @@ if ($ViServersList.count -gt 0) { # } Write-Host "$((Get-Date).ToString("o")) [INFO] Writing ESX Inventory file ..." $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force -ErrorAction Stop + $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] ESX Inventory issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -423,9 +451,23 @@ if ($ViServersList.count -gt 0) { } if ($ViDatastoresInfos) { + + if ($(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv")) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViDsInventory.*.csv files ..." + $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} + If ($ExtraCsvFiles) { + try { + Get-ChildItem $ExtraCsvFiles.name | Remove-Item -Force -Confirm:$false + } catch { + AltAndCatchFire "Cannot remove extra csv files" + } + } + } + try { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Datastore Inventory files ..." $ViDatastoresInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViDsInventory.csv -Force -ErrorAction Stop + $ViDatastoresInfos|Out-File -Path /mnt/wfs/inventory/ViDsInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] Datastore Export-Csv issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" From 8e6e463bdbf58fb5e04cee64b53b83d5e6808452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 5 Oct 2023 00:38:14 +0200 Subject: [PATCH 146/201] Update VbrPullStatistics.ps1 #275 --- opt/veeam/VbrPullStatistics.ps1 | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index e4842198..ab511132 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.35" +$ScriptVersion = "0.9.36" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -391,9 +391,24 @@ if ($($VbrJobsStates.data)) { Write-Host "$((Get-Date).ToString("o")) [EROR] VbrVmInventory import issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" } + + if ($(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv")) { + Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating VbrVmInventory.*.csv files ..." + $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} + If ($ExtraCsvFiles) { + try { + Get-ChildItem $ExtraCsvFiles.name | Remove-Item -Force -Confirm:$false + } catch { + AltAndCatchFire "Cannot remove extra csv files" + } + } + } + Write-Host "$((Get-Date).ToString("o")) [INFO] Import/Export VBR inventories ..." try { - Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Sort-Object LastRestorePoint -Descending | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force + $VbrVmInventoryCsv = Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Sort-Object LastRestorePoint -Descending + $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force + $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -ErrorAction Stop -Force } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] Import/Export VBR inventories issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" From 05a5f8e1b1cd61d6db244bfd1f5fb0840d5987b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 10 Oct 2023 01:20:23 +0200 Subject: [PATCH 147/201] Update ViPullStatistics.ps1 #358 --- opt/sexigraf/ViPullStatistics.ps1 | 360 +++++++++++++++--------------- 1 file changed, 183 insertions(+), 177 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 2dddaa5d..6c1be791 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -1,8 +1,8 @@ #!/usr/bin/pwsh -Command # -param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) +param([parameter (Mandatory=$true)] [string] $Server, [parameter (Mandatory=$true)] [string] $SessionFile, [parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1029" +$ScriptVersion = "0.9.1031" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -10,18 +10,29 @@ $ExecStart = $(Get-Date).ToUniversalTime() $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" (Get-Process -Id $pid).PriorityClass = 'Idle' +$SexiMutex = New-Object System.Threading.Mutex($false, "SexiMutex") +# https://learn-powershell.net/2014/09/30/using-mutexes-to-write-data-to-the-same-logfile-across-processes-with-powershell/ +# https://github.com/ryan-leap/GreenMeansGoMutexDemo + +function SexiLogger { + param($Text2Log) + Write-Host "$((Get-Date).ToString("o")) $Text2Log" + $null = $SexiMutex.WaitOne(500) + Add-Content -Path "/var/log/sexigraf/ViPullStatistics.log" -Value "$((Get-Date).ToString("o")) $($Server) $Text2Log" + $SexiMutex.ReleaseMutex() +} function AltAndCatchFire { - Param($ExitReason) - Write-Host "$((Get-Date).ToString("o")) [EROR] $ExitReason" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" - Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + param($ExitReason) + SexiLogger "[EROR] $ExitReason" + SexiLogger "[EROR] $($Error[0])" + SexiLogger "[EROR] Exit" Stop-Transcript exit } function GetRootDc { - Param($child_object) + param($child_object) if ($xfolders_vcenter_parent_h[$child_object.Parent.value]) { $Parent_folder = $child_object.Parent.value while ($xfolders_vcenter_type_h[$xfolders_vcenter_parent_h[$Parent_folder]] -notmatch "^Datacenter$") { @@ -32,7 +43,7 @@ function GetRootDc { } function NameCleaner { - Param($NameToClean) + param($NameToClean) $NameToClean = $NameToClean -replace "[ .]","_" [System.Text.NormalizationForm]$NormalizationForm = "FormD" $NameToClean = $NameToClean.Normalize($NormalizationForm) @@ -171,20 +182,15 @@ function MultiQueryPerf300 { try { Start-Transcript -Path "/var/log/sexigraf/ViPullStatistics.$($Server).log" -Append -Confirm:$false -Force -UseMinimalHeader - Start-Transcript -Path "/var/log/sexigraf/ViPullStatistics.log" -Append -Confirm:$false -Force -UseMinimalHeader - Write-Host "$((Get-Date).ToString("o")) [INFO] ViPullStatistics v$ScriptVersion" - if ($vSanPull = Test-Path -Path $("/etc/cron.d/vsan_" + $Server.Replace(".","_"))) { - Start-Transcript -Path "/var/log/sexigraf/VsanDisksPullStatistics.$($Server).log" -Append -Confirm:$false -Force -UseMinimalHeader - Start-Transcript -Path "/var/log/sexigraf/VsanDisksPullStatistics.log" -Append -Confirm:$false -Force -UseMinimalHeader - } + SexiLogger "[INFO] ViPullStatistics v$ScriptVersion" } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ViPullStatistics logging failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + SexiLogger "[EROR] ViPullStatistics logging failure" + SexiLogger "[EROR] Exit" exit } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Importing PowerCli and Graphite PowerShell modules ..." + SexiLogger "[INFO] Importing PowerCli and Graphite PowerShell modules ..." Import-Module VMware.VimAutomation.Common, VMware.VimAutomation.Core, VMware.VimAutomation.Sdk, VMware.VimAutomation.Storage $PowerCliConfig = Set-PowerCLIConfiguration -ProxyPolicy NoProxy -DefaultVIServerMode Single -InvalidCertificateAction Ignore -ParticipateInCeip:$false -DisplayDeprecationWarnings:$false -Confirm:$false -Scope Session Import-Module -Name /usr/local/share/powershell/Modules/Graphite-PowerShell-Functions/Graphite-Powershell.psm1 -Global -Force -SkipEditionCheck @@ -193,15 +199,15 @@ try { } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for another ViPullStatistics for $Server ..." + SexiLogger "[INFO] Looking for another ViPullStatistics for $Server ..." $DupViPullStatisticsProcess = Get-PSHostProcessInfo|%{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}|?{$_ -match "ViPullStatistics" -and $_ -match "$Server"} # https://github.com/PowerShell/PowerShell/issues/13944 if (($DupViPullStatisticsProcess|Measure-Object).Count -gt 1) { $DupViPullStatisticsProcessId = (Get-PSHostProcessInfo|?{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '|?{$_ -match "$Server"}}).ProcessId[0] $DupViPullStatisticsProcessTime = [INT32](ps -p $DupViPullStatisticsProcessId -o etimes).split()[-1] if ($DupViPullStatisticsProcessTime -gt 300) { - Write-Host "$((Get-Date).ToString("o")) [WARN] ViPullStatistics for $Server is already running for more than 5 minutes!" - Write-Host "$((Get-Date).ToString("o")) [WARN] Killing stunned ViPullStatistics for $Server" + SexiLogger "[WARN] ViPullStatistics for $Server is already running for more than 5 minutes!" + SexiLogger "[WARN] Killing stunned ViPullStatistics for $Server" Stop-Process -Id $DupViPullStatisticsProcessId -Force } else { AltAndCatchFire "ViPullStatistics for $Server is already running!" @@ -214,16 +220,16 @@ try { if ($SessionFile) { try { $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop - Write-Host "$((Get-Date).ToString("o")) [INFO] SessionToken found in SessionFile, attempting connection to $Server ..." + SexiLogger "[INFO] SessionToken found in SessionFile, attempting connection to $Server ..." # https://zhengwu.org/validating-connection-result-of-connect-viserver/ $ServerConnection = Connect-VIServer -Server $Server -Session $SessionToken -Force -ErrorAction Stop if ($ServerConnection.IsConnected) { # $PwCliContext = Get-PowerCLIContext - Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to vCenter $($ServerConnection.Name) version $($ServerConnection.Version) build $($ServerConnection.Build)" + SexiLogger "[INFO] Connected to vCenter $($ServerConnection.Name) version $($ServerConnection.Version) build $($ServerConnection.Build)" } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] SessionToken not found, invalid or connection failure" - Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting explicit connection ..." + SexiLogger "[WARN] SessionToken not found, invalid or connection failure" + SexiLogger "[WARN] Attempting explicit connection ..." } if (!$($global:DefaultVIServer)) { @@ -243,7 +249,7 @@ if ($SessionFile) { $ServerConnection = Connect-VIServer -Server $Server -User $CredStoreLogin -Password $CredStorePassword -Force -ErrorAction Stop if ($ServerConnection.IsConnected) { # $PwCliContext = Get-PowerCLIContext - Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to vCenter $($ServerConnection.Name) version $($ServerConnection.Version) build $($ServerConnection.Build)" + SexiLogger "[INFO] Connected to vCenter $($ServerConnection.Name) version $($ServerConnection.Version) build $($ServerConnection.Build)" $SessionSecretName = "vmw_" + $Server.Replace(".","_") + ".key" $ServerConnection.SessionSecret | Out-File -FilePath /tmp/$SessionSecretName -Force } @@ -257,7 +263,7 @@ if ($SessionFile) { try { if ($($global:DefaultVIServer)) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing vCenter/ESX $Server ..." + SexiLogger "[INFO] Start processing vCenter/ESX $Server ..." $ServiceInstance = Get-View ServiceInstance -Server $Server # $ServiceManager = Get-View $ServiceInstance.Content.serviceManager -property "" -Server $Server $ServiceInstanceServerClock = $ServiceInstance.CurrentTime() @@ -271,7 +277,7 @@ try { try { if ($ServiceInstance) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing SessionManager & EventManager ..." + SexiLogger "[INFO] Processing SessionManager & EventManager ..." try { $AuthorizationManager = Get-View $ServiceInstance.Content.AuthorizationManager -Server $Server $group_d1 = Get-View $ServiceInstance.Content.RootFolder @@ -281,7 +287,7 @@ try { if ($UserRole.Privilege -match "Sessions.TerminateSession") { $SessionManager = Get-View $ServiceInstance.Content.SessionManager -Property SessionList -Server $Server # Permission to perform this operation was denied. Required privilege 'Sessions.TerminateSession' on managed object with id 'Folder-group-d1'. } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] Sessions.TerminateSession privilege not detected, SessionManager skipped ..." + SexiLogger "[INFO] Sessions.TerminateSession privilege not detected, SessionManager skipped ..." } $EventManager = Get-View $ServiceInstance.Content.EventManager -Property latestEvent, description -Server $Server } catch { @@ -291,7 +297,7 @@ try { AltAndCatchFire "ServiceInstance check failure" } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] $($Error[0])" } try { @@ -319,7 +325,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_version = NameCleaner $($ServiceInstance.Content.About.Version + "." + $ServiceInstance.Content.About.Build) $vmware_version_h["vi.$vcenter_name.vi.version.vpx.product.$vcenter_version"] ++ - Write-Host "$((Get-Date).ToString("o")) [INFO] vCenter objects collect ..." + SexiLogger "[INFO] vCenter objects collect ..." try { $vcenter_folders = Get-View -ViewType Folder -Property Name, Parent -Server $Server @@ -334,7 +340,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { AltAndCatchFire "Get-View failure" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Building objects tables ..." + SexiLogger "[INFO] Building objects tables ..." $vcenter_resource_pools_h = @{} $vcenter_resource_pools_owner_vms_h = @{} @@ -440,7 +446,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $script:xfolders_vcenter_parent_h = @{} $script:xfolders_vcenter_type_h = @{} - Write-Host "$((Get-Date).ToString("o")) [INFO] vCenter objects relationship discover ..." + SexiLogger "[INFO] vCenter objects relationship discover ..." foreach ($vcenter_xfolder in [array]$vcenter_datacenters + [array]$vcenter_folders + [array]$vcenter_clusters + [array]$vcenter_resource_pools + [array]$vcenter_vmhosts + [array]$vcenter_pods) { try { @@ -450,7 +456,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } catch {} } - Write-Host "$((Get-Date).ToString("o")) [INFO] Performance metrics collection ..." + SexiLogger "[INFO] Performance metrics collection ..." $HostMultiMetrics = @( "net.bytesRx.average", @@ -475,18 +481,18 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if (($vcenter_vmhosts|Measure-Object).Count -gt 0) { try { $HostMultiStatsTime = Measure-Command {$HostMultiStats = MultiQueryPerfAll $($vcenter_vmhosts.moref) $HostMultiMetrics} - Write-Host "$((Get-Date).ToString("o")) [INFO] All hosts multi metrics collected in $($HostMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + SexiLogger "[INFO] All hosts multi metrics collected in $($HostMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { AltAndCatchFire "ESX MultiQueryPerfAll failure" } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] No ESX in vCenter $vcenter_name" + SexiLogger "[INFO] No ESX in vCenter $vcenter_name" } if (($vcenter_vms|Measure-Object).Count -eq 0) { - Write-Host "$((Get-Date).ToString("o")) [INFO] No VMs in vCenter $vcenter_name" + SexiLogger "[INFO] No VMs in vCenter $vcenter_name" } elseif (($vcenter_vms|Measure-Object).Count -gt 10000) { - Write-Host "$((Get-Date).ToString("o")) [INFO] 10K+ VMs mode for vCenter $vcenter_name" + SexiLogger "[INFO] 10K+ VMs mode for vCenter $vcenter_name" $VmMultiMetricsR1 = @( "cpu.ready.summation", @@ -497,7 +503,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $VmMultiStatsTime = Measure-Command {$VmMultiStats = MultiQueryPerf $($vcenter_vms.moref) $VmMultiMetricsR1} - Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics 1st round collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + SexiLogger "[INFO] All vms multi metrics 1st round collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { AltAndCatchFire "VM MultiQueryPerf R1 failure" } @@ -511,7 +517,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerf $($vcenter_vms.moref) $VmMultiMetricsR2} - Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics 2nd round collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + SexiLogger "[INFO] All vms multi metrics 2nd round collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { AltAndCatchFire "VM MultiQueryPerf R2 failure" } @@ -525,7 +531,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # try { # $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} - # Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + # SexiLogger "[INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" # } catch { # AltAndCatchFire "VM MultiQueryPerfAll failure" # } @@ -544,7 +550,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $VmMultiStatsTime = Measure-Command {$VmMultiStats = MultiQueryPerf $($vcenter_vms.moref) $VmMultiMetrics} - Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + SexiLogger "[INFO] All vms multi metrics collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { AltAndCatchFire "VM MultiQueryPerf failure" } @@ -556,7 +562,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} - Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + SexiLogger "[INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { AltAndCatchFire "VM MultiQueryPerfAll failure" } @@ -570,16 +576,16 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { ) try { $ClusterMultiStatsTime = Measure-Command {$ClusterMultiStats = MultiQueryPerf300 $($vcenter_clusters_h.Values.moref) $ClusterMultiMetrics} - Write-Host "$((Get-Date).ToString("o")) [INFO] All Clusters multi metrics collected in $($ClusterMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + SexiLogger "[INFO] All Clusters multi metrics collected in $($ClusterMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" - Write-Host "$((Get-Date).ToString("o")) [EROR] VM MultiQueryPerf failure" + SexiLogger "[EROR] $($Error[0])" + SexiLogger "[EROR] VM MultiQueryPerf failure" } } if ($ServiceInstance.Content.About.ApiVersion -ge 6.7) { if ($vcenter_vmhosts_NodeUuid_name_h.Count -gt 0 -and $vSanPull) { - Write-Host "$((Get-Date).ToString("o")) [INFO] vCenter ApiVersion is 6.7+ so we can call vSAN API" + SexiLogger "[INFO] vCenter ApiVersion is 6.7+ so we can call vSAN API" $VsanPerformanceManager = Get-VSANView -Id VsanPerformanceManager-vsan-performance-manager -Server $Server $VsanClusterHealthSystem = Get-VSANView -Id VsanVcClusterHealthSystem-vsan-cluster-health-system -Server $Server $VsanSpaceReportSystem = Get-VSANView -Id VsanSpaceReportSystem-vsan-cluster-space-report-system -Server $Server @@ -587,7 +593,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # VsanClusterHealthSummary/VsanPhysicalDiskHealthSummary better than VsanManagedDisksInfo since we can query at the cluster level # if ($vcenter_clusters_vsan_efa_h.keys) { # $VsanVcDiskManagementSystem = Get-VSANView -Id VimClusterVsanVcDiskManagementSystem-vsan-disk-management-system -Server $Server - # Write-Host "$((Get-Date).ToString("o")) [INFO] vSAN EFA clusters detected ..." + # SexiLogger "[INFO] vSAN EFA clusters detected ..." # } } } @@ -598,7 +604,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster = $vcenter_clusters_h[$vcenter_cluster_moref] $vcenter_cluster_name = nameCleaner $vcenter_cluster.Name $vcenter_cluster_dc_name = nameCleaner $(getRootDc $vcenter_cluster) - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name in datacenter $vcenter_cluster_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name in datacenter $vcenter_cluster_dc_name" } catch { AltAndCatchFire "cluster name cleaning issue" } @@ -631,11 +637,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name quickstats collect issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Cluster $vcenter_cluster_name quickstats collect issue" + SexiLogger "[EROR] $($Error[0])" } } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name root resource pool not found ?!" + SexiLogger "[EROR] Cluster $vcenter_cluster_name root resource pool not found ?!" } if ($vcenter_cluster.summary) { @@ -646,11 +652,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.cpu.total", $vcenter_cluster.summary.totalCpu) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numVmotions", $vcenter_cluster.summary.numVmotions) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name summary collect issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Cluster $vcenter_cluster_name summary collect issue" + SexiLogger "[EROR] $($Error[0])" } } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name summary missing " + SexiLogger "[EROR] Cluster $vcenter_cluster_name summary missing " } try { @@ -662,8 +668,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.numXVMotions", $ClusterMultiStats[$PerfCounterTable["vmop.numXVMotion.latest"]][$vcenter_cluster.moref.value][""]) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name xyzMotion collect issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Cluster $vcenter_cluster_name xyzMotion collect issue" + SexiLogger "[EROR] $($Error[0])" } try { @@ -698,11 +704,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.quickstats.drs.DrsScore", $vcenter_cluster.summary.DrsScore) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Cluster $vcenter_cluster_name DrsScore collect issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Cluster $vcenter_cluster_name DrsScore collect issue" + SexiLogger "[EROR] $($Error[0])" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name hosts in datacenter $vcenter_cluster_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name hosts in datacenter $vcenter_cluster_dc_name" $vcenter_cluster_hosts_pcpus = 0 $vcenter_cluster_hosts_vms_moref = @() @@ -761,8 +767,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host sensors issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host sensors issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -805,8 +811,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name network metrics issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name network metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -823,8 +829,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name hba metrics issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name hba metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -832,8 +838,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h["vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.PathState.$($vcenter_cluster_host_path.PathState)"] ++ } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name MultipathState issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name MultipathState issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -845,8 +851,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats power metrics issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name fatstats power metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -857,8 +863,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats cpu.totalCapacity metrics issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name fatstats cpu.totalCapacity metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -869,8 +875,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats mem.totalCapacity metrics issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name fatstats mem.totalCapacity metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -881,8 +887,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats cpu.latency metrics issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name fatstats cpu.latency metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -893,16 +899,16 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.overallStatus", $vcenter_cluster_host_overallStatus) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name fatstats overallStatus metrics issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name fatstats overallStatus metrics issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedCpuFairness", $vcenter_cluster_host.summary.quickStats.distributedCpuFairness) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.distributedMemoryFairness", $vcenter_cluster_host.summary.quickStats.distributedMemoryFairness) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name Distributed Fairness quickstats issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name Distributed Fairness quickstats issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -911,8 +917,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$vcenter_cluster_host_name.quickstats.Uptime", $vcenter_cluster_host.summary.quickStats.uptime) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_cluster_host_name overall quickstats issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_cluster_host_name overall quickstats issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } } @@ -938,7 +944,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.power", $vcenter_cluster_hosts_power_usage) } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name vms in datacenter $vcenter_cluster_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name vms in datacenter $vcenter_cluster_dc_name" $vcenter_cluster_vms_vcpus = 0 $vcenter_cluster_vms_vram = 0 @@ -1026,16 +1032,16 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.storage.delta", $vcenter_cluster_vm_snap_size) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $vcenter_cluster_vm_name snapshot compute issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $vcenter_cluster_vm_name snapshot compute issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } try { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.storage.committed", $vcenter_cluster_vm.summary.storage.committed) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.vm.$vcenter_cluster_vm_name.storage.uncommitted", $vcenter_cluster_vm.summary.storage.uncommitted) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $vcenter_cluster_vm_name storage commit metric issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $vcenter_cluster_vm_name storage commit metric issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } if ($vcenter_cluster_vm.summary.runtime.powerState -eq "poweredOn") { @@ -1156,7 +1162,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.runtime.vm.on", $vcenter_cluster_vms_on) $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.runtime.vm.dead", $vcenter_cluster_hosts_vms_dead) - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name datastores in datacenter $vcenter_cluster_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name cluster $vcenter_cluster_name datastores in datacenter $vcenter_cluster_dc_name" $vcenter_cluster_datastores_count = 0 $vcenter_cluster_datastores_capacity = 0 @@ -1198,8 +1204,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_datastores_uncommitted += $vcenter_cluster_datastore_uncommitted } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] datastore processing issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] datastore processing issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } if ($vcenter_cluster_datastore.summary.type -notmatch "vsan") { @@ -1221,8 +1227,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.datastore.$vcenter_cluster_datastore_name.iorm.datastoreIops", $vcenter_cluster_datastore_iops) $vcenter_cluster_datastores_iops += $vcenter_cluster_datastore_iops } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive performance metrics for datastore $vcenter_cluster_datastore_name in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive performance metrics for datastore $vcenter_cluster_datastore_name in cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } } else { # VsanPullStatistics @@ -1232,7 +1238,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($vcenter_cluster_datastore_vsan_cluster_uuid.replace("-","") -match $vcenter_cluster_datastore_vsan_uuid.replace("-","") -and $vSanPull -and $ServiceInstance.Content.About.ApiVersion -ge 6.7) { # skip vSAN HCI Mesh - Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VsanPerfQuery in cluster $vcenter_cluster_name (v6.7+) ..." + SexiLogger "[INFO] Start processing VsanPerfQuery in cluster $vcenter_cluster_name (v6.7+) ..." # https://vdc-download.vmware.com/vmwb-repository/dcr-public/bd51bfdb-3107-4a66-9f63-30aa3fae196e/98507723-ab67-4908-88fa-7c99e0743f0f/vim.cluster.VsanPerformanceManager.html#queryVsanPerf # MethodInvocationException: Exception calling "VsanPerfQueryPerf" with "2" argument(s): "Invalid Argument. Only one wildcard query allowed in query specs." # Config.VsanHostConfig.ClusterInfo.NodeUuid # $VsanPerformanceManager.VsanPerfQueryStatsObjectInformation($vcenter_cluster.moref).VsanHealth @@ -1253,14 +1259,14 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # $VsanHostsAndClusterPerfQuerySpec += New-Object VMware.Vsan.Views.VsanPerfQuerySpec -property @{entityRefId="vsan-vnic-net:*";labels=@("rxThroughput","txThroughput","rxPackets","txPackets","tcpSackRcvBlocksRate","txPacketsLossRate","tcpSackRexmitsRate","rxPacketsLossRate","tcpHalfopenDropRate","tcpRcvoopackRate","tcpRcvduppackRate","tcpRcvdupackRate","tcpTimeoutDropRate","tcpTxRexmitRate","tcpRxErrRate","tcpSackSendBlocksRate");startTime=$ServiceInstanceServerClock_5;endTime=$ServiceInstanceServerClock} # "arpDropRate" if ($vcenter_clusters_vsan_efa_h[$vcenter_cluster_moref]) { # EFA vSan cluster - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing EFA vSAN metrics in cluster $vcenter_cluster_name (v8.0+) ..." + SexiLogger "[INFO] Processing EFA vSAN metrics in cluster $vcenter_cluster_name (v8.0+) ..." # VsanClusterHealthSummary/VsanPhysicalDiskHealthSummary better than VsanManagedDisksInfo since we can query at the cluster level # $VsanHostsAndClusterStoragePoolSpec = New-Object VMware.Vsan.Views.VimVsanHostQueryVsanDisksSpec -property @{vsanDiskType="storagePool"} try { $ClusterHealthSummary = $VsanClusterHealthSystem.VsanQueryVcClusterHealthSummary($vcenter_cluster.moref,$null,$null,$false,"physicalDisksHealth",$true,$null,$null,$null) } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive VsanQueryVcClusterHealthSummary in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive VsanQueryVcClusterHealthSummary in cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } if ($ClusterHealthSummary.PhysicalDisksHealth) { @@ -1280,7 +1286,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $VsanHostsAndClusterPerfQuerySpec += New-Object VMware.Vsan.Views.VsanPerfQuerySpec -property @{entityRefId="vsan-esa-disk-scsifw:$PhysicalDisksHealthVsanUuid";labels=@("latencyDevRead","latencyDevWrite");startTime=$ServiceInstanceServerClock_5;endTime=$ServiceInstanceServerClock} } } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] Empty vSAN PhysicalDisksHealth in cluster $vcenter_cluster_name" + SexiLogger "[WARN] Empty vSAN PhysicalDisksHealth in cluster $vcenter_cluster_name" } } else { # Non EFA vSAN cluster # cache-disk disk-group @@ -1296,11 +1302,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $VsanHostsAndClusterPerfQueryTime = Measure-Command {$VsanHostsAndClusterPerfQuery = $VsanPerformanceManager.VsanPerfQueryPerf($VsanHostsAndClusterPerfQuerySpec,$vcenter_cluster.moref)} - Write-Host "$((Get-Date).ToString("o")) [INFO] VsanPerfQueryPerf metrics collected in $($VsanHostsAndClusterPerfQueryTime.TotalSeconds) sec for vSAN Cluster $vcenter_cluster_name in vCenter $vcenter_name" + SexiLogger "[INFO] VsanPerfQueryPerf metrics collected in $($VsanHostsAndClusterPerfQueryTime.TotalSeconds) sec for vSAN Cluster $vcenter_cluster_name in vCenter $vcenter_name" } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive VsanPerfQuery in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive VsanPerfQuery in cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } if ($VsanHostsAndClusterPerfQuery) { @@ -1466,11 +1472,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] Empty VsanPerfQuery in cluster $vcenter_cluster_name" + SexiLogger "[WARN] Empty VsanPerfQuery in cluster $vcenter_cluster_name" } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing SmartStatsSummary in cluster $vcenter_cluster_name (v6.7+) ..." + SexiLogger "[INFO] Start processing SmartStatsSummary in cluster $vcenter_cluster_name (v6.7+) ..." # https://www.virtuallyghetto.com/2017/04/getting-started-wthe-new-powercli-6-5-1-get-vsanview-cmdlet.html # https://github.com/lamw/vghetto-scripts/blob/master/powershell/VSANSmartsData.ps1 $VcClusterSmartStatsSummary = $VsanClusterHealthSystem.VsanQueryVcClusterSmartStatsSummary($vcenter_cluster.moref) @@ -1480,20 +1486,20 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { foreach ($SmartStatsEsxDisk in $SmartStatsEsx.SmartStats) { $SmartStatsEsxDiskName = NameCleaner $SmartStatsEsxDisk.Disk foreach ($SmartStatsEsxDiskStats in $SmartStatsEsxDisk.Stats|?{$_.Value -ne $null}) { - if ($SmartStatsEsxDiskStats.Parameter -and !$vcenter_cluster_h["vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$SmartStatsEsxName.vsan.disk.smart.$SmartStatsEsxDiskName.$($SmartStatsEsxDiskStats.Parameter)"]) { - $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$SmartStatsEsxName.vsan.disk.smart.$SmartStatsEsxDiskName.$($SmartStatsEsxDiskStats.Parameter)", $($SmartStatsEsxDiskStats.Value)) + if ($SmartStatsEsxDiskStats.parameter -and !$vcenter_cluster_h["vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$SmartStatsEsxName.vsan.disk.smart.$SmartStatsEsxDiskName.$($SmartStatsEsxDiskStats.parameter)"]) { + $vcenter_cluster_h.add("vsan.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.esx.$SmartStatsEsxName.vsan.disk.smart.$SmartStatsEsxDiskName.$($SmartStatsEsxDiskStats.parameter)", $($SmartStatsEsxDiskStats.Value)) } } } } } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive VcClusterSmartStatsSummary in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive VcClusterSmartStatsSummary in cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing spaceUsageByObjectType in vSAN cluster $vcenter_cluster_name (v6.2+) ..." + SexiLogger "[INFO] Processing spaceUsageByObjectType in vSAN cluster $vcenter_cluster_name (v6.2+) ..." $ClusterVsanSpaceUsageReport = $VsanSpaceReportSystem.VsanQuerySpaceUsage($vcenter_cluster.Moref) $ClusterVsanSpaceUsageReportObjList = $ClusterVsanSpaceUsageReport.spaceDetail.spaceUsageByObjectType @@ -1522,12 +1528,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # } # } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive VsanQuerySpaceUsage for cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive VsanQuerySpaceUsage for cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing SyncingVsanObjectsSummary in cluster $vcenter_cluster_name (v6.7+) ..." + SexiLogger "[INFO] Start processing SyncingVsanObjectsSummary in cluster $vcenter_cluster_name (v6.7+) ..." # https://vdc-download.vmware.com/vmwb-repository/dcr-public/b21ba11d-4748-4796-97e2-7000e2543ee1/b4a40704-fbca-4222-902c-2500f5a90f3f/vim.cluster.VsanObjectSystem.html#querySyncingVsanObjectsSummary # https://vdc-download.vmware.com/vmwb-repository/dcr-public/9ab58fbf-b389-4e15-bfd4-a915910be724/7872dcb2-3287-40e1-ba00-71071d0e19ff/vim.vsan.VsanSyncReason.html $QuerySyncingVsanObjectsSummary = $VsanObjectSystem.QuerySyncingVsanObjectsSummary($vcenter_cluster.Moref,$(New-Object VMware.Vsan.Views.VsanSyncingObjectFilter -property @{NumberOfObjects="200"})) @@ -1551,12 +1557,12 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive SyncingVsanObjectsSummary in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive SyncingVsanObjectsSummary in cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VsanObjectIdentityAndHealth in cluster $vcenter_cluster_name ..." + SexiLogger "[INFO] Start processing VsanObjectIdentityAndHealth in cluster $vcenter_cluster_name ..." $vcenter_cluster_ObjectIdentities = $VsanObjectSystem.VsanQueryObjectIdentities($vcenter_cluster.moref,$null,$null,$true,$false,$false) ### TODO optimize if ($vcenter_cluster_ObjectIdentities.Health.ObjectHealthDetail) { foreach ($ObjectHealth in $vcenter_cluster_ObjectIdentities.Health.ObjectHealthDetail) { @@ -1566,8 +1572,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Unable to retreive VsanObjectIdentityAndHealth from cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [WARN] $($Error[0])" + SexiLogger "[WARN] Unable to retreive VsanObjectIdentityAndHealth from cluster $vcenter_cluster_name" + SexiLogger "[WARN] $($Error[0])" } } } @@ -1586,7 +1592,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_cluster_h.add("vmw.$vcenter_name.$vcenter_cluster_dc_name.$vcenter_cluster_name.superstats.datastore.iops", $vcenter_cluster_datastores_iops) } - Write-Host "$((Get-Date).ToString("o")) [INFO] Sending cluster, hosts, vms and datastores metrics to Graphite for cluster $vcenter_cluster_name ..." + SexiLogger "[INFO] Sending cluster, hosts, vms and datastores metrics to Graphite for cluster $vcenter_cluster_name ..." Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vcenter_cluster_h -DateTime $ExecStart } @@ -1596,7 +1602,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $vcenter_pod_name = NameCleaner $vcenter_pod.Name $vcenter_pod_dc_name = nameCleaner $(getRootDc $vcenter_pod) - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name pod $vcenter_pod_name in datacenter $vcenter_pod_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name pod $vcenter_pod_name in datacenter $vcenter_pod_dc_name" } catch { AltAndCatchFire "pod name cleaning issue" } @@ -1631,7 +1637,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } $vcenter_standalone_host_dc_name = nameCleaner $(getRootDc $vcenter_standalone_pool) - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name in datacenter $vcenter_standalone_host_dc_name" } catch { AltAndCatchFire "standalone_host name cleaning issue" } @@ -1660,8 +1666,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_standalone_host sensors issue in datacenter $vcenter_standalone_host_dc_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_standalone_host sensors issue in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[EROR] $($Error[0])" } if ($vcenter_standalone_host.overallStatus) { @@ -1680,11 +1686,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.quickstats.cpu.total", $vcenter_standalone_pool.summary.totalCpu) $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.quickstats.overallStatus", $vcenter_standalone_host_overallStatus) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_standalone_host_name quickstats issue in datacenter $vcenter_standalone_host_dc_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_standalone_host_name quickstats issue in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[EROR] $($Error[0])" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name datastores in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name datastores in datacenter $vcenter_standalone_host_dc_name" foreach ($vcenter_standalone_host_datastore in $vcenter_datastores_h[$vcenter_standalone_pool.Datastore.value]|?{$_}) { if ($vcenter_standalone_host_datastore.summary.accessible) { @@ -1713,8 +1719,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.datastore.$vcenter_standalone_host_datastore_name.iorm.datastoreIops", $vcenter_standalone_host_datastore_iops) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] datastore processing issue on ESX $vcenter_standalone_host_name in datacenter $vcenter_standalone_host_dc_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] datastore processing issue on ESX $vcenter_standalone_host_name in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[EROR] $($Error[0])" } } } @@ -1734,8 +1740,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_standalone_host_name network metrics issue in datacenter $vcenter_standalone_host_dc_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_standalone_host_name network metrics issue in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -1751,8 +1757,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_standalone_host_name hba metrics issue in datacenter $vcenter_standalone_host_dc_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_standalone_host_name hba metrics issue in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[EROR] $($Error[0])" } try { @@ -1760,11 +1766,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_standalone_host_h["esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.superstats.PathState.$($vcenter_standalone_host_path.PathState)"] ++ } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ESX $vcenter_standalone_host_name MultipathState issue in cluster $vcenter_cluster_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ESX $vcenter_standalone_host_name MultipathState issue in cluster $vcenter_cluster_name" + SexiLogger "[EROR] $($Error[0])" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name vms in datacenter $vcenter_standalone_host_dc_name" + SexiLogger "[INFO] Processing vCenter $vcenter_name standalone host $vcenter_standalone_host_name vms in datacenter $vcenter_standalone_host_dc_name" $StandaloneResourcePoolPrivateMemory = 0 $StandaloneResourcePoolSharedMemory = 0 @@ -1860,16 +1866,16 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.storage.delta", $vcenter_standalone_host_vm_snap_size) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $vcenter_standalone_host_vm_name snapshot compute issue standalone host $vcenter_standalone_host_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $vcenter_standalone_host_vm_name snapshot compute issue standalone host $vcenter_standalone_host_name" + SexiLogger "[EROR] $($Error[0])" } try { $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.storage.committed", $vcenter_standalone_host_vm.summary.storage.committed) $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.storage.uncommitted", $vcenter_standalone_host_vm.summary.storage.uncommitted) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $vcenter_standalone_host_vm_name storage commit metric issue standalone host $vcenter_standalone_host_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $vcenter_standalone_host_vm_name storage commit metric issue standalone host $vcenter_standalone_host_name" + SexiLogger "[EROR] $($Error[0])" } if ($vcenter_standalone_host_vm.summary.runtime.powerState -eq "poweredOn") { @@ -2009,7 +2015,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } if ($SessionManager) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name SessionList" + SexiLogger "[INFO] Processing vCenter $vcenter_name SessionList" $vcenter_session_list_h = @{} $SessionList = $SessionManager.sessionList if ($SessionList) { @@ -2022,7 +2028,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing vCenter $vcenter_name events" + SexiLogger "[INFO] Processing vCenter $vcenter_name events" $vcenter_events_h = @{} $vCenterFilteredEventTypeId = [System.Collections.ArrayList]@() @@ -2043,8 +2049,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $null = $vCenterFilteredEventTypeIdCat.add($vCenterEventInfo.key) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] vCenter $vcenter_name EventInfo collect issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] vCenter $vcenter_name EventInfo collect issue" + SexiLogger "[EROR] $($Error[0])" } } @@ -2058,8 +2064,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vCenterEventsHistoryCollector = $EventManager.QueryEvents($vCenterEventFilterSpec) $vCenterEventsHistoryCollectorCat = $EventManager.QueryEvents($vCenterEventFilterSpecCat) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] vCenter $vcenter_name EventManager QueryEvents issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] vCenter $vcenter_name EventManager QueryEvents issue" + SexiLogger "[EROR] $($Error[0])" } if ($vCenterEventsHistoryCollector -or $vCenterEventsHistoryCollectorCat) { @@ -2091,10 +2097,10 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vcenter_events_h -DateTime $ExecStart } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] vCenter $vcenter_name no new event collected" + SexiLogger "[INFO] vCenter $vcenter_name no new event collected" } } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] No EventInfo to process in vCenter $vcenter_name" + SexiLogger "[EROR] No EventInfo to process in vCenter $vcenter_name" } } @@ -2102,11 +2108,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vmware_version_h -DateTime $ExecStart - Write-Host "$((Get-Date).ToString("o")) [INFO] End processing vCenter $vcenter_name" + SexiLogger "[INFO] End processing vCenter $vcenter_name" } elseif ($ServiceInstance.Content.About.ApiType -match "HostAgent") { - Write-Host "$((Get-Date).ToString("o")) [INFO] HostAgent detected, collecting unmanaged objects ..." + SexiLogger "[INFO] HostAgent detected, collecting unmanaged objects ..." $vcenter_name = "_unmanaged_" $unmanaged_host_dc_name = "_unmanaged_" @@ -2124,7 +2130,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { AltAndCatchFire "Get-View failure" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Performance metrics collection ..." + SexiLogger "[INFO] Performance metrics collection ..." $HostMultiMetrics = @( "net.bytesRx.average", @@ -2139,7 +2145,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { try { $HostMultiStatsTime = Measure-Command {$HostMultiStats = MultiQueryPerfAll $($vcenter_vmhosts.moref) $HostMultiMetrics} - Write-Host "$((Get-Date).ToString("o")) [INFO] All hosts multi metrics collected in $($HostMultiStatsTime.TotalSeconds) sec for Unmanaged ESX $esx_name" + SexiLogger "[INFO] All hosts multi metrics collected in $($HostMultiStatsTime.TotalSeconds) sec for Unmanaged ESX $esx_name" } catch { AltAndCatchFire "ESX MultiQueryPerfAll failure" } @@ -2163,14 +2169,14 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($vcenter_vms) { try { $VmMultiStatsTime = Measure-Command {$VmMultiStats = MultiQueryPerf $($vcenter_vms.moref) $VmMultiMetrics} - Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics collected in $($VmMultiStatsTime.TotalSeconds) sec for Unmanaged ESX $esx_name" + SexiLogger "[INFO] All vms multi metrics collected in $($VmMultiStatsTime.TotalSeconds) sec for Unmanaged ESX $esx_name" } catch { AltAndCatchFire "VM MultiQueryPerf failure" } try { $VmMultiStatsTime = Measure-Command {$VmMultiStats += MultiQueryPerfAll $($vcenter_vms.moref) $VmMultiMetricsAll} - Write-Host "$((Get-Date).ToString("o")) [INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" + SexiLogger "[INFO] All vms multi metrics instanced collected in $($VmMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { AltAndCatchFire "VM MultiQueryPerfAll failure" } @@ -2185,7 +2191,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $unmanaged_host_name = NameCleaner $unmanaged_host.name ### previously vmk0 ip cleaned } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing Unmanaged ESX $esx_name" + SexiLogger "[INFO] Processing Unmanaged ESX $esx_name" } catch { AltAndCatchFire "Unmanaged ESX name cleaning issue" } @@ -2214,8 +2220,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name sensors issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name sensors issue" + SexiLogger "[EROR] $($Error[0])" } if ($unmanaged_host.overallStatus.value__) { @@ -2234,11 +2240,11 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.cpu.total", $unmanaged_compute_resource.summary.totalCpu) $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.overallStatus", $unmanaged_host_overallStatus) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name quickstats issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name quickstats issue" + SexiLogger "[EROR] $($Error[0])" } - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing Unmanaged ESX $esx_name datastores" + SexiLogger "[INFO] Processing Unmanaged ESX $esx_name datastores" foreach ($unmanaged_host_datastore in $vcenter_datastores) { if ($unmanaged_host_datastore.summary.accessible) { @@ -2267,8 +2273,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.datastore.$unmanaged_host_datastore_name.iorm.datastoreIops", $unmanaged_host_datastore_iops) # } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] datastore processing issue on Unmanaged ESX $esx_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] datastore processing issue on Unmanaged ESX $esx_name" + SexiLogger "[EROR] $($Error[0])" } } } @@ -2288,8 +2294,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name network metrics issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name network metrics issue" + SexiLogger "[EROR] $($Error[0])" } try { @@ -2305,8 +2311,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name hba metrics issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name hba metrics issue" + SexiLogger "[EROR] $($Error[0])" } $UnamagedResourcePoolPrivateMemory = 0 @@ -2412,16 +2418,16 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.storage.delta", $unmanaged_host_vm_snap_size) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $unmanaged_host_vm_name snapshot compute issue standalone host $unmanaged_host_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $unmanaged_host_vm_name snapshot compute issue standalone host $unmanaged_host_name" + SexiLogger "[EROR] $($Error[0])" } try { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.storage.committed", $unmanaged_host_vm.summary.storage.committed) $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.storage.uncommitted", $unmanaged_host_vm.summary.storage.uncommitted) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $unmanaged_host_vm_name storage commit metric issue standalone host $unmanaged_host_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $unmanaged_host_vm_name storage commit metric issue standalone host $unmanaged_host_name" + SexiLogger "[EROR] $($Error[0])" } if ($unmanaged_host_vm.summary.runtime.powerState -eq "poweredOn") { @@ -2516,8 +2522,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($unmanaged_host_vm.summary.quickStats.ConsumedOverheadMemory -gt 0) {$UnamagedResourcePoolConsumedOverheadMemory += $unmanaged_host_vm.summary.quickStats.ConsumedOverheadMemory} } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VM $unmanaged_host_vm_name metric issue on unmanaged host $unmanaged_host_name" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VM $unmanaged_host_vm_name metric issue on unmanaged host $unmanaged_host_name" + SexiLogger "[EROR] $($Error[0])" } } elseif ($unmanaged_host_vm.summary.runtime.powerState -eq "poweredOff") { @@ -2539,13 +2545,13 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.runtime.vm.dead", $unmanaged_host_vms_dead) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name quickstats issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name quickstats issue" + SexiLogger "[EROR] $($Error[0])" } Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $unmanaged_host_h -DateTime $ExecStart - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing Unmanaged ESX $esx_name events" + SexiLogger "[INFO] Processing Unmanaged ESX $esx_name events" $vcenter_events_h = @{} $vCenterFilteredEventTypeId = @() @@ -2561,8 +2567,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $null = $vCenterFilteredEventTypeIdCat.add($vCenterEventInfo.key) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name EventInfo collect issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name EventInfo collect issue" + SexiLogger "[EROR] $($Error[0])" } } @@ -2583,8 +2589,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $vCenterEventsHistoryCollectorCat = $vCenterEventsHistoryCollectorCatObj.ReadNextEvents("999") } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Unmanaged ESX $esx_name EventManager QueryEvents issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Unmanaged ESX $esx_name EventManager QueryEvents issue" + SexiLogger "[EROR] $($Error[0])" } if ($vCenterEventsHistoryCollector -or $vCenterEventsHistoryCollectorCat) { @@ -2611,10 +2617,10 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vcenter_events_h -DateTime $ExecStart } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] Unmanaged ESX $esx_name no new event collected" + SexiLogger "[INFO] Unmanaged ESX $esx_name no new event collected" } } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] No EventInfo to process in Unmanaged ESX $esx_name" + SexiLogger "[EROR] No EventInfo to process in Unmanaged ESX $esx_name" } } @@ -2622,7 +2628,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $vmware_version_h -DateTime $ExecStart - Write-Host "$((Get-Date).ToString("o")) [INFO] End processing Unmanaged ESX $esx_name" + SexiLogger "[INFO] End processing Unmanaged ESX $esx_name" } else { AltAndCatchFire "$Server is not a vCenter/ESXi!" From 92dd4d762ccf9f49ff1dc6f825423bf80ca294e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 10 Oct 2023 01:20:29 +0200 Subject: [PATCH 148/201] Update showlog.php --- var/www/admin/showlog.php | 1 - 1 file changed, 1 deletion(-) diff --git a/var/www/admin/showlog.php b/var/www/admin/showlog.php index 8b862123..de43ca67 100644 --- a/var/www/admin/showlog.php +++ b/var/www/admin/showlog.php @@ -256,7 +256,6 @@ function updateLog() { $tail = new PHPTail(array( "VI Offline Inventory" => "/var/log/sexigraf/ViOfflineInventory.log", - "vSAN Statistics" => "/var/log/sexigraf/VsanDisksPullStatistics.log", "VI Statistics" => "/var/log/sexigraf/ViPullStatistics.log", "VBR Statistics" => "/var/log/sexigraf/VbrPullStatistics.log", "xMotion Garbage Collector" => "/var/log/sexigraf/xMotionGarbageCollector.log", From 7efb873f24f6c77287a0ead2dbb3e6493186f52f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 10 Oct 2023 16:05:18 +0200 Subject: [PATCH 149/201] Update ViOfflineInventory --- etc/cron.d/ViOfflineInventory | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/cron.d/ViOfflineInventory b/etc/cron.d/ViOfflineInventory index 464a65dd..579e658f 100644 --- a/etc/cron.d/ViOfflineInventory +++ b/etc/cron.d/ViOfflineInventory @@ -1 +1 @@ -3 */1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/ViOfflineInventory.ps1 -credstore /mnt/wfs/inventory/vipscredentials.xml >/dev/null 2>&1 +3 */1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/ViOfflineInventory.ps1 -credstore /mnt/wfs/inventory/vipscredentials.xml >/dev/null 2>&1 \ No newline at end of file From f2f1c649eea9e7a9ec5623ae53a5422a940ce532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 10 Oct 2023 16:05:21 +0200 Subject: [PATCH 150/201] Update importSexiGrafBundle.sh --- var/www/scripts/importSexiGrafBundle.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/var/www/scripts/importSexiGrafBundle.sh b/var/www/scripts/importSexiGrafBundle.sh index dfc155ae..617e3818 100644 --- a/var/www/scripts/importSexiGrafBundle.sh +++ b/var/www/scripts/importSexiGrafBundle.sh @@ -71,7 +71,10 @@ if [ -d "/mnt/wfs/whisper" ]; then # /bin/sed -i 's/--sessionfile \/tmp\/vpx_/-sessionfile \/tmp\/vmw_/g' /etc/cron.d/vsan_* # /bin/sed -i 's/\.dat$/.key >\/dev\/null 2\>\&1/g' /etc/cron.d/vsan_* - echo "# Virtual SAN has left the building" | tee /etc/cron.d/vsan_* > /dev/null # TODO deal with /etc/cron.d/vsan_* + if compgen -G "/etc/cron.d/vsan_*" > /dev/null; then + echo "# Virtual SAN has left the building" | tee /etc/cron.d/vsan_* + fi + # TODO rm -rf /mnt/wfs/whisper/vsan/*/*/*/esx/vsan /bin/sed -i 's/\/usr\/bin\/perl \/root\/ViPullStatistics\.pl --credstore \/var\/www\/\.vmware\/credstore\/vicredentials\.xml --server/\/usr\/bin\/pwsh -NonInteractive -NoProfile -f \/opt\/sexigraf\/ViPullStatistics\.ps1 -credstore \/mnt\/wfs\/inventory\/vipscredentials\.xml -server/g' /etc/cron.d/vi_* From 8b4b82a964c1df2f5404bf1425efe4880fe96c5d Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 12 Oct 2023 11:29:28 +0200 Subject: [PATCH 151/201] Update ViPullStatistics.ps1 #358 --- opt/sexigraf/ViPullStatistics.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 6c1be791..26d93176 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([parameter (Mandatory=$true)] [string] $Server, [parameter (Mandatory=$true)] [string] $SessionFile, [parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1031" +$ScriptVersion = "0.9.1032" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -10,16 +10,16 @@ $ExecStart = $(Get-Date).ToUniversalTime() $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" (Get-Process -Id $pid).PriorityClass = 'Idle' -$SexiMutex = New-Object System.Threading.Mutex($false, "SexiMutex") +$SexiViMutex = New-Object System.Threading.Mutex($false, "SexiViMutex") # https://learn-powershell.net/2014/09/30/using-mutexes-to-write-data-to-the-same-logfile-across-processes-with-powershell/ # https://github.com/ryan-leap/GreenMeansGoMutexDemo function SexiLogger { param($Text2Log) Write-Host "$((Get-Date).ToString("o")) $Text2Log" - $null = $SexiMutex.WaitOne(500) + $null = $SexiViMutex.WaitOne(500) Add-Content -Path "/var/log/sexigraf/ViPullStatistics.log" -Value "$((Get-Date).ToString("o")) $($Server) $Text2Log" - $SexiMutex.ReleaseMutex() + $SexiViMutex.ReleaseMutex() } function AltAndCatchFire { From 74238cc7f63fca7cd85db2bca9ccafe9b332bdf3 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 12 Oct 2023 15:28:48 +0200 Subject: [PATCH 152/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 160 +++++++++++++++++--------------- 1 file changed, 85 insertions(+), 75 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index ab511132..92a0860e 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.36" +$ScriptVersion = "0.9.37" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -10,12 +10,23 @@ $ExecStart = $(Get-Date).ToUniversalTime() $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" (Get-Process -Id $pid).PriorityClass = 'Idle' +$SexiVbrMutex = New-Object System.Threading.Mutex($false, "SexiVbrMutex") +# https://learn-powershell.net/2014/09/30/using-mutexes-to-write-data-to-the-same-logfile-across-processes-with-powershell/ +# https://github.com/ryan-leap/GreenMeansGoMutexDemo + +function SexiLogger { + param($Text2Log) + SexiLogger "$Text2Log" + $null = $SexiVbrMutex.WaitOne(500) + Add-Content -Path "/var/log/sexigraf/VbrPullStatistics.log" -Value "$((Get-Date).ToString("o")) $($Server) $Text2Log" + $SexiVbrMutex.ReleaseMutex() +} function AltAndCatchFire { Param($ExitReason) - Write-Host "$((Get-Date).ToString("o")) [EROR] $ExitReason" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" - Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + SexiLogger "[EROR] $ExitReason" + SexiLogger "[EROR] $($Error[0])" + SexiLogger "[EROR] Exit" Stop-Transcript exit } @@ -32,31 +43,30 @@ function NameCleaner { try { Start-Transcript -Path "/var/log/sexigraf/VbrPullStatistics.$($Server).log" -Append -Confirm:$false -Force -UseMinimalHeader - Start-Transcript -Path "/var/log/sexigraf/VbrPullStatistics.log" -Append -Confirm:$false -Force -UseMinimalHeader - Write-Host "$((Get-Date).ToString("o")) [INFO] VbrPullStatistics v$ScriptVersion" + SexiLogger "[INFO] VbrPullStatistics v$ScriptVersion" } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VbrPullStatistics logging failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] Exit" + SexiLogger "[EROR] VbrPullStatistics logging failure" + SexiLogger "[EROR] Exit" exit } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Importing Graphite PowerShell module ..." + SexiLogger "[INFO] Importing Graphite PowerShell module ..." Import-Module -Name /usr/local/share/powershell/Modules/Graphite-PowerShell-Functions/Graphite-Powershell.psm1 -Global -Force -SkipEditionCheck } catch { AltAndCatchFire "Powershell modules import failure" } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] Looking for another VbrPullStatistics for $Server ..." + SexiLogger "[INFO] Looking for another VbrPullStatistics for $Server ..." $DupVbrPullStatisticsProcess = Get-PSHostProcessInfo|%{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}|?{$_ -match "VbrPullStatistics" -and $_ -match "$Server"} # https://github.com/PowerShell/PowerShell/issues/13944 if (($DupVbrPullStatisticsProcess|Measure-Object).Count -gt 1) { $DupVbrPullStatisticsProcessId = (Get-PSHostProcessInfo|?{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '|?{$_ -match "$Server"}}).ProcessId[0] $DupVbrPullStatisticsProcessTime = [INT32](ps -p $DupVbrPullStatisticsProcessId -o etimes).split()[-1] if ($DupVbrPullStatisticsProcessTime -gt 300) { - Write-Host "$((Get-Date).ToString("o")) [WARN] VbrPullStatistics for $Server is already running for more than 5 minutes!" - Write-Host "$((Get-Date).ToString("o")) [WARN] Killing stunned VbrPullStatistics for $Server" + SexiLogger "[WARN] VbrPullStatistics for $Server is already running for more than 5 minutes!" + SexiLogger "[WARN] Killing stunned VbrPullStatistics for $Server" Stop-Process -Id $DupVbrPullStatisticsProcessId -Force } else { AltAndCatchFire "VbrPullStatistics for $Server is already running!" @@ -71,23 +81,23 @@ if ($SessionFile) { $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" if ([DateTime]$(Get-Content -Path /tmp/$SessionSecretExpiration) -gt $ExecStart.AddMinutes(5)) { $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop - Write-Host "$((Get-Date).ToString("o")) [INFO] SessionToken found in SessionFile, attempting connection to $Server ..." + SexiLogger "[INFO] SessionToken found in SessionFile, attempting connection to $Server ..." $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders if ($($VbrJobsStates.data)) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" + SexiLogger "[INFO] Connected to VBR REST API Server $Server" } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] Connection failure or no job state" + SexiLogger "[WARN] Connection failure or no job state" } } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] Token has expired or is about to ..." + SexiLogger "[WARN] Token has expired or is about to ..." } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] SessionToken or SessionSecretExpiration not found, invalid or connection failure" + SexiLogger "[WARN] SessionToken or SessionSecretExpiration not found, invalid or connection failure" } if (!$($VbrJobsStates.data)) { - Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting token refresh ..." + SexiLogger "[WARN] Attempting token refresh ..." try { $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" $SessionRefresh = Get-Content -Path /tmp/$SessionRefreshPath -ErrorAction Stop @@ -104,20 +114,20 @@ if ($SessionFile) { $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders if (!$($VbrJobsStates.data)) { - Write-Host "$((Get-Date).ToString("o")) [WARN] Token refresh failed!" - Write-Host "$((Get-Date).ToString("o")) [WARN] Known issue on v11!" + SexiLogger "[WARN] Token refresh failed!" + SexiLogger "[WARN] Known issue on v11!" # https://forums.veeam.com/restful-api-f30/how-to-handle-the-refresh-token-t74916.html } } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] Token refresh failed!" + SexiLogger "[WARN] Token refresh failed!" } } catch { - Write-Host "$((Get-Date).ToString("o")) [WARN] Token refresh issue!" + SexiLogger "[WARN] Token refresh issue!" } } if (!$($VbrJobsStates.data)) { - Write-Host "$((Get-Date).ToString("o")) [WARN] Attempting explicit connection ..." + SexiLogger "[WARN] Attempting explicit connection ..." try { $createstorexml = New-Object -TypeName XML $createstorexml.Load($credstore) @@ -136,7 +146,7 @@ if ($SessionFile) { $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders if ($($VbrJobsStates.data)) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Connected to VBR REST API Server $Server" + SexiLogger "[INFO] Connected to VBR REST API Server $Server" $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" @@ -162,7 +172,7 @@ if ($($VbrJobsStates.data)) { $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} $VbrDataTable = @{} $vbrserver_name = NameCleaner $Server - Write-Host "$((Get-Date).ToString("o")) [INFO] Start processing VBR Server $Server ..." + SexiLogger "[INFO] Start processing VBR Server $Server ..." $VbrJobsStatesTable = @{} foreach ($VbrJobState in $VbrJobsStates.data) { @@ -183,15 +193,15 @@ if ($($VbrJobsStates.data)) { } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] VBR repositories states collect ..." + SexiLogger "[INFO] VBR repositories states collect ..." $VbrRepositoriesStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupInfrastructure/repositories/states") -Headers $VbrAuthHeaders } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] repositories states collect failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] repositories states collect failure" + SexiLogger "[EROR] $($Error[0])" } if ($VbrRepositoriesStates.data) { - Write-Host "$((Get-Date).ToString("o")) [INFO] VBR repositories processing ..." + SexiLogger "[INFO] VBR repositories processing ..." $VbrRepositoryTable = @{} foreach ($VbrRepository in $VbrRepositoriesStates.data) { $VbrRepositoryName = NameCleaner $VbrRepository.name @@ -206,19 +216,19 @@ if ($($VbrJobsStates.data)) { } catch {} } } else { - Write-Host "$((Get-Date).ToString("o")) [WARN] No repositories ?!" + SexiLogger "[WARN] No repositories ?!" } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] VBR ScaleOutRepositories collect ..." + SexiLogger "[INFO] VBR ScaleOutRepositories collect ..." $VbrScaleOutRepositories = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupInfrastructure/scaleOutRepositories") -Headers $VbrAuthHeaders } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ScaleOutRepositories collect failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ScaleOutRepositories collect failure" + SexiLogger "[EROR] $($Error[0])" } if ($VbrScaleOutRepositories.data) { - Write-Host "$((Get-Date).ToString("o")) [INFO] SOBR performanceTier processing ..." + SexiLogger "[INFO] SOBR performanceTier processing ..." foreach ($VbrScaleOutRepository in $VbrScaleOutRepositories.data) { try { $VbrScaleOutRepositoryName = NameCleaner $VbrScaleOutRepository.name @@ -243,28 +253,28 @@ if ($($VbrJobsStates.data)) { $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.RealUsedPct"] = 100 } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Issue processing SOBR $($VbrScaleOutRepository.name)" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Issue processing SOBR $($VbrScaleOutRepository.name)" + SexiLogger "[EROR] $($Error[0])" } } } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] No SOBR" + SexiLogger "[INFO] No SOBR" } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] VBR 5min old objectRestorePoints collect ..." + SexiLogger "[INFO] VBR 5min old objectRestorePoints collect ..." # $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders $VbrObjectRestorePoints5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/objectRestorePoints?platformNameFilter=VmWare&limit=999&createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] objectRestorePoints collect failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] objectRestorePoints collect failure" + SexiLogger "[EROR] $($Error[0])" } # $VbrVmwareServers = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/inventory/vmware/hosts") -Headers $VbrAuthHeaders if ($VbrObjectRestorePoints5.data) { - # Write-Host "$((Get-Date).ToString("o")) [INFO] VBR backupObjects collect ..." + # SexiLogger "[INFO] VBR backupObjects collect ..." # $VbrBackupObjects5 = @{} # foreach ($VbrObjectRestorePoint5 in $VbrObjectRestorePoints5.data) { # try { @@ -273,16 +283,16 @@ if ($($VbrJobsStates.data)) { # $VbrBackupObjects5.Add($VbrBackupObjects.data[0].name,$VbrBackupObjects[0].data) # } # } catch { - # Write-Host "$((Get-Date).ToString("o")) [EROR] backupObjects collect failure" - # Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + # SexiLogger "[EROR] backupObjects collect failure" + # SexiLogger "[EROR] $($Error[0])" # } # } # Too Slow !!! try { - Write-Host "$((Get-Date).ToString("o")) [INFO] VBR backupObjects collect ..." + SexiLogger "[INFO] VBR backupObjects collect ..." $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&limit=9999") -Headers $VbrAuthHeaders # TODO limit=9999+ ? if ($VbrBackupObjects.data) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Building VBR backupObjects table ..." + SexiLogger "[INFO] Building VBR backupObjects table ..." $VbrBackupObjectsTable = @{} foreach ($VbrBackupObject in $VbrBackupObjects.data) { try { @@ -291,13 +301,13 @@ if ($($VbrJobsStates.data)) { } } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] backupObjects collect failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] backupObjects collect failure" + SexiLogger "[EROR] $($Error[0])" } $VbrObjectRestorePoints5SessionsId = $VbrObjectRestorePoints5.data.backupId|Select-Object -Unique - Write-Host "$((Get-Date).ToString("o")) [INFO] VBR Sessions collect ..." + SexiLogger "[INFO] VBR Sessions collect ..." $VbrSessions5 = @{} foreach ($VbrObjectRestorePoints5SessionId in $VbrObjectRestorePoints5SessionsId) { try { @@ -306,18 +316,18 @@ if ($($VbrJobsStates.data)) { $VbrSessions5.Add($VbrObjectRestorePoints5SessionId,$VbrObjectRestorePoints5Session) } } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] backupObjects collect failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] backupObjects collect failure" + SexiLogger "[EROR] $($Error[0])" } } if (Test-Path /mnt/wfs/inventory/ViVmInventory.csv) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Importing VM inventory ..." + SexiLogger "[INFO] Importing VM inventory ..." try { $ViVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] ViVmInventory import issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] ViVmInventory import issue" + SexiLogger "[EROR] $($Error[0])" } if ($ViVmInventory) { @@ -329,12 +339,12 @@ if ($($VbrJobsStates.data)) { } if (Test-Path /mnt/wfs/inventory/VbrInv_$($Server).csv) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Importing $($Server) VBR inventory ..." + SexiLogger "[INFO] Importing $($Server) VBR inventory ..." try { $VbrVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/VbrInv_$($Server).csv -ErrorAction Stop) } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VbrVmInventory import issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VbrVmInventory import issue" + SexiLogger "[EROR] $($Error[0])" } if ($ViVmInventory) { $VbrVmInventoryTable = @{} @@ -350,7 +360,7 @@ if ($($VbrJobsStates.data)) { $VbrVmInventoryTable = @{} } - Write-Host "$((Get-Date).ToString("o")) [INFO] Building $($Server) VBR DataTable and inventory ..." + SexiLogger "[INFO] Building $($Server) VBR DataTable and inventory ..." foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { if ($VbrJobsStatesTable[$VbrSessions5[$VbrObjectRestorePoint.backupId].jobId]) { $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name @@ -384,16 +394,16 @@ if ($($VbrJobsStates.data)) { } if ($VbrVmInventoryTable.Values) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Exporting $($Server) VBR inventory ..." + SexiLogger "[INFO] Exporting $($Server) VBR inventory ..." try { $VbrVmInventoryTable.Values|Sort-Object VM|Export-Csv /mnt/wfs/inventory/VbrInv_$($Server).csv -ErrorAction Stop -Force } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] VbrVmInventory import issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] VbrVmInventory import issue" + SexiLogger "[EROR] $($Error[0])" } if ($(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv")) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating VbrVmInventory.*.csv files ..." + SexiLogger "[INFO] Rotating VbrVmInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { @@ -404,36 +414,36 @@ if ($($VbrJobsStates.data)) { } } - Write-Host "$((Get-Date).ToString("o")) [INFO] Import/Export VBR inventories ..." + SexiLogger "[INFO] Import/Export VBR inventories ..." try { $VbrVmInventoryCsv = Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Sort-Object LastRestorePoint -Descending $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -ErrorAction Stop -Force } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Import/Export VBR inventories issue" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Import/Export VBR inventories issue" + SexiLogger "[EROR] $($Error[0])" } } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] No VbrVmInventoryTable values ?!" + SexiLogger "[EROR] No VbrVmInventoryTable values ?!" } } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] ViVmInventory import issue" + SexiLogger "[EROR] ViVmInventory import issue" } } else { - Write-Host "$((Get-Date).ToString("o")) [EROR] No ViVmInventory" + SexiLogger "[EROR] No ViVmInventory" } } try { - Write-Host "$((Get-Date).ToString("o")) [INFO] VBR ended sessions collect ..." + SexiLogger "[INFO] VBR ended sessions collect ..." $VbrEndedSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?endedAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders } catch { - Write-Host "$((Get-Date).ToString("o")) [EROR] Sessions collect failure" - Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" + SexiLogger "[EROR] Sessions collect failure" + SexiLogger "[EROR] $($Error[0])" } if ($VbrEndedSessions.data) { - Write-Host "$((Get-Date).ToString("o")) [INFO] Processing VBR ended sessions ..." + SexiLogger "[INFO] Processing VBR ended sessions ..." foreach ($VbrEndedSession in $VbrEndedSessions.data) { $job_name = NameCleaner $VbrEndedSession.name if ($VbrEndedSession.result.result -eq "Success") { @@ -450,14 +460,14 @@ if ($($VbrJobsStates.data)) { } $VbrDataTable["veeam.vbr.$vbrserver_name.exec.duration"] = $($(Get-Date).ToUniversalTime() - $ExecStart).TotalSeconds - Write-Host "$((Get-Date).ToString("o")) [INFO] Sending veeam data to Graphite for VBR server $Server ..." + SexiLogger "[INFO] Sending veeam data to Graphite for VBR server $Server ..." Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrDataTable -DateTime $ExecStart - Write-Host "$((Get-Date).ToString("o")) [INFO] End of VBR server $server processing ..." + SexiLogger "[INFO] End of VBR server $server processing ..." } else { - Write-Host "$((Get-Date).ToString("o")) [INFO] No Running Jobs on VBR server $server ..." - Write-Host "$((Get-Date).ToString("o")) [INFO] Exit" + SexiLogger "[INFO] No Running Jobs on VBR server $server ..." + SexiLogger "[INFO] Exit" Stop-Transcript exit } \ No newline at end of file From 5cf91128893a7c30110060c98b841748072794f1 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 12 Oct 2023 23:05:45 +0200 Subject: [PATCH 153/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 92a0860e..47f242ec 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -16,7 +16,7 @@ $SexiVbrMutex = New-Object System.Threading.Mutex($false, "SexiVbrMutex") function SexiLogger { param($Text2Log) - SexiLogger "$Text2Log" + Write-Host "$((Get-Date).ToString("o")) $Text2Log" $null = $SexiVbrMutex.WaitOne(500) Add-Content -Path "/var/log/sexigraf/VbrPullStatistics.log" -Value "$((Get-Date).ToString("o")) $($Server) $Text2Log" $SexiVbrMutex.ReleaseMutex() From 26beb09016c85191a8e2e6be7d5b9a85331cd057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 3 Nov 2023 00:21:10 +0100 Subject: [PATCH 154/201] Update graphite.conf 275 --- etc/apache2/sites-available/graphite.conf | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/etc/apache2/sites-available/graphite.conf b/etc/apache2/sites-available/graphite.conf index 16e32e4e..dbd7f29f 100644 --- a/etc/apache2/sites-available/graphite.conf +++ b/etc/apache2/sites-available/graphite.conf @@ -125,6 +125,25 @@ ExtendedStatus On RewriteRule .* - [F] + ProxyPass /sexihistory/ ! + Alias /sexihistory/ /mnt/wfs/inventory/ + + RewriteEngine On + RewriteCond %{HTTP_REFERER} /sexigraf-web-admin [NC,OR] + RewriteCond %{HTTP_REFERER} /sexiadmin/ [NC] + RewriteRule .* - [L] + RewriteRule .* - [F] + Options Indexes + AllowOverride All + Require all granted + Allow from all + + + + Order Allow,Deny + Deny from all + + ProxyPass /favicon.ico ! Alias /favicon.ico /var/www/admin/images/favicon.ico From eb46ddd970a50f3d428eb2d889f2976fad17a126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 3 Nov 2023 00:21:20 +0100 Subject: [PATCH 155/201] Update index.php #275 --- var/www/admin/index.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/admin/index.php b/var/www/admin/index.php index 2d148da9..e14bc36c 100644 --- a/var/www/admin/index.php +++ b/var/www/admin/index.php @@ -75,7 +75,7 @@
    -
    Refresh Inventories
    +
    Inventory Refresh & History
    From f8b8b1750952b675ed009653d46fa5939280cc94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 3 Nov 2023 00:21:25 +0100 Subject: [PATCH 156/201] Update refresh-inventory.php #275 --- var/www/admin/refresh-inventory.php | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/var/www/admin/refresh-inventory.php b/var/www/admin/refresh-inventory.php index 7bab6ebf..d96dcbd4 100644 --- a/var/www/admin/refresh-inventory.php +++ b/var/www/admin/refresh-inventory.php @@ -1,6 +1,6 @@ @@ -55,6 +55,20 @@ '; } ?> + +'.$filename.''; + } + } +?> +

    Inventory History:

    +
    + From 4f2e58dff55033b882084dfe89ed3116214315c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 7 Nov 2023 23:48:28 +0100 Subject: [PATCH 157/201] Update ViOfflineInventory --- etc/cron.d/ViOfflineInventory | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/cron.d/ViOfflineInventory b/etc/cron.d/ViOfflineInventory index 579e658f..464a65dd 100644 --- a/etc/cron.d/ViOfflineInventory +++ b/etc/cron.d/ViOfflineInventory @@ -1 +1 @@ -3 */1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/ViOfflineInventory.ps1 -credstore /mnt/wfs/inventory/vipscredentials.xml >/dev/null 2>&1 \ No newline at end of file +3 */1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/ViOfflineInventory.ps1 -credstore /mnt/wfs/inventory/vipscredentials.xml >/dev/null 2>&1 From 387d5b017f53d937d35d0ebd10cc9fcb53213010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 7 Nov 2023 23:48:31 +0100 Subject: [PATCH 158/201] Update xMotionGarbageCollector --- etc/cron.d/xMotionGarbageCollector | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/cron.d/xMotionGarbageCollector b/etc/cron.d/xMotionGarbageCollector index 297c04f8..99f91d18 100644 --- a/etc/cron.d/xMotionGarbageCollector +++ b/etc/cron.d/xMotionGarbageCollector @@ -1 +1 @@ -37 1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/xMotionGarbageCollector.ps1 >/dev/null 2>&1 \ No newline at end of file +37 1 * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/xMotionGarbageCollector.ps1 >/dev/null 2>&1 From ab0c432df2b59e03651564dd43d0f5ff565edf23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 7 Nov 2023 23:48:39 +0100 Subject: [PATCH 159/201] Update refresh-inventory.php --- var/www/admin/refresh-inventory.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/admin/refresh-inventory.php b/var/www/admin/refresh-inventory.php index d96dcbd4..c741494f 100644 --- a/var/www/admin/refresh-inventory.php +++ b/var/www/admin/refresh-inventory.php @@ -59,7 +59,7 @@ '.$filename.''; From b8440e3ebb7223b20b70dec89d606fff83d3436c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 7 Nov 2023 23:48:47 +0100 Subject: [PATCH 160/201] Update ViOfflineInventory.ps1 DayOfWeek -match "Monday" --- opt/sexigraf/ViOfflineInventory.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index bbb161fe..3e296f94 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -362,7 +362,7 @@ if ($ViServersList.count -gt 0) { if ($ViVmsInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv")) { + if ($(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday") { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViVmInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { @@ -407,7 +407,7 @@ if ($ViServersList.count -gt 0) { if ($ViEsxsInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv")) { + if ($(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday") { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViEsxInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { @@ -452,7 +452,7 @@ if ($ViServersList.count -gt 0) { if ($ViDatastoresInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv")) { + if ($(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday") { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViDsInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { From fcfb2294196ed4c526663a4b22930043ff1070a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 7 Nov 2023 23:54:29 +0100 Subject: [PATCH 161/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 47f242ec..03bd0fbd 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.37" +$ScriptVersion = "0.9.38" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -402,7 +402,7 @@ if ($($VbrJobsStates.data)) { SexiLogger "[EROR] $($Error[0])" } - if ($(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv")) { + if ($(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { SexiLogger "[INFO] Rotating VbrVmInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { From 4149bb33c08e2492282cc652f1d0fbf538dd3bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 7 Nov 2023 23:54:33 +0100 Subject: [PATCH 162/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 3e296f94..a1c1145b 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.83" +$ScriptVersion = "0.9.84" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -362,7 +362,7 @@ if ($ViServersList.count -gt 0) { if ($ViVmsInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday") { + if ($(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViVmInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { @@ -407,7 +407,7 @@ if ($ViServersList.count -gt 0) { if ($ViEsxsInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday") { + if ($(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViEsxInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { @@ -452,7 +452,7 @@ if ($ViServersList.count -gt 0) { if ($ViDatastoresInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday") { + if ($(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViDsInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { From cc7ef908ecf6646c29f29f1d64e6c8c97ce9c6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 11 Nov 2023 00:04:27 +0100 Subject: [PATCH 163/201] Update vbrcredstore.php --- var/www/admin/vbrcredstore.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/admin/vbrcredstore.php b/var/www/admin/vbrcredstore.php index a72fcac5..eb107f12 100644 --- a/var/www/admin/vbrcredstore.php +++ b/var/www/admin/vbrcredstore.php @@ -96,7 +96,7 @@ exec("/usr/bin/pwsh -NonInteractive -NoProfile -f /opt/veeam/VbrConnect.ps1 -server " . escapeshellcmd($_POST["input-vbr"]) . " -username " . escapeshellcmd($_POST["input-username"]) . " -password " . escapeshellcmd($_POST["input-password"]), $null, $return_var); if ($return_var) { $errorHappened = true; - $errorMessage = "Wrong username/password or no answer at TCP:9419"; + $errorMessage = "Wrong username/password, AccessDenied or no answer at TCP:9419"; } } From a0baf51917fe91f9bf1a5ad11ea47b758396bdae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 11 Nov 2023 00:04:57 +0100 Subject: [PATCH 164/201] Update VbrPullStatistics.ps1 275 --- opt/veeam/VbrPullStatistics.ps1 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 03bd0fbd..ab8acd95 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.38" +$ScriptVersion = "0.9.39" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -402,7 +402,7 @@ if ($($VbrJobsStates.data)) { SexiLogger "[EROR] $($Error[0])" } - if ($(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + if ($(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv")) { SexiLogger "[INFO] Rotating VbrVmInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { @@ -418,7 +418,9 @@ if ($($VbrJobsStates.data)) { try { $VbrVmInventoryCsv = Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Sort-Object LastRestorePoint -Descending $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force - $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -ErrorAction Stop -Force + if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -ErrorAction Stop -Force + } } catch { SexiLogger "[EROR] Import/Export VBR inventories issue" SexiLogger "[EROR] $($Error[0])" From a13c24b1bc2825d88c62f828d67328b7938af797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 11 Nov 2023 00:04:59 +0100 Subject: [PATCH 165/201] Update ViOfflineInventory.ps1 275 --- opt/sexigraf/ViOfflineInventory.ps1 | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index a1c1145b..67277d5f 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.84" +$ScriptVersion = "0.9.85" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -362,7 +362,7 @@ if ($ViServersList.count -gt 0) { if ($ViVmsInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + if ($(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv")) { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViVmInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { @@ -398,7 +398,9 @@ if ($ViServersList.count -gt 0) { # } Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory file ..." $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.csv -Force -ErrorAction Stop - $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop + if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop + } } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] VM Inventory issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -407,7 +409,7 @@ if ($ViServersList.count -gt 0) { if ($ViEsxsInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + if ($(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv")) { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViEsxInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { @@ -443,7 +445,9 @@ if ($ViServersList.count -gt 0) { # } Write-Host "$((Get-Date).ToString("o")) [INFO] Writing ESX Inventory file ..." $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force -ErrorAction Stop - $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop + if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop + } } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] ESX Inventory issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" @@ -452,7 +456,7 @@ if ($ViServersList.count -gt 0) { if ($ViDatastoresInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") -and $ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + if ($(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv")) { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViDsInventory.*.csv files ..." $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { @@ -467,7 +471,9 @@ if ($ViServersList.count -gt 0) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Datastore Inventory files ..." $ViDatastoresInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViDsInventory.csv -Force -ErrorAction Stop - $ViDatastoresInfos|Out-File -Path /mnt/wfs/inventory/ViDsInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop + if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + $ViDatastoresInfos|Out-File -Path /mnt/wfs/inventory/ViDsInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop + } } catch { Write-Host "$((Get-Date).ToString("o")) [EROR] Datastore Export-Csv issue" Write-Host "$((Get-Date).ToString("o")) [EROR] $($Error[0])" From 6fe15a60626002d1ccabab06022822154850e70e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 11 Nov 2023 00:10:15 +0100 Subject: [PATCH 166/201] Update exportSexiGrafBundle.sh #254 --- var/www/scripts/exportSexiGrafBundle.sh | 26 ++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/var/www/scripts/exportSexiGrafBundle.sh b/var/www/scripts/exportSexiGrafBundle.sh index d661c3ae..3782f7a2 100644 --- a/var/www/scripts/exportSexiGrafBundle.sh +++ b/var/www/scripts/exportSexiGrafBundle.sh @@ -34,19 +34,19 @@ if [ -d "/mnt/wfs/whisper" ]; then cp /mnt/wfs/inventory/*.csv /mnt/wfs/sexigraf-dump/ # Retrieve credential store items - if [ -a "/var/www/.vmware/credstore/vicredentials.xml" ]; then - for creditem in $(/usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml list | egrep -v "Server|^$" | sed "s/[[:space:]]\+/;/") - do - vcenter=$(echo $creditem | cut -d ";" -f 1) - username=$(echo $creditem | cut -d ";" -f 2) - password="/usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml get --server $vcenter --username '$username'" - password=$(eval "$password" | cut -c 11-) - echo "$vcenter;$username;$password" >> /mnt/wfs/sexigraf-dump/conf/vicredentials.conf - done - # File encoding - openssl des3 -salt -in /mnt/wfs/sexigraf-dump/conf/vicredentials.conf -out /mnt/wfs/sexigraf-dump/conf/vicredentials.conf.ss -pass pass:sexigraf - rm -f /mnt/wfs/sexigraf-dump/conf/vicredentials.conf - fi + # if [ -a "/var/www/.vmware/credstore/vicredentials.xml" ]; then + # for creditem in $(/usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml list | egrep -v "Server|^$" | sed "s/[[:space:]]\+/;/") + # do + # vcenter=$(echo $creditem | cut -d ";" -f 1) + # username=$(echo $creditem | cut -d ";" -f 2) + # password="/usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml get --server $vcenter --username '$username'" + # password=$(eval "$password" | cut -c 11-) + # echo "$vcenter;$username;$password" >> /mnt/wfs/sexigraf-dump/conf/vicredentials.conf + # done + # # File encoding + # openssl des3 -salt -in /mnt/wfs/sexigraf-dump/conf/vicredentials.conf -out /mnt/wfs/sexigraf-dump/conf/vicredentials.conf.ss -pass pass:sexigraf + # rm -f /mnt/wfs/sexigraf-dump/conf/vicredentials.conf + # fi if [ -a "/mnt/wfs/inventory/vipscredentials.xml" ]; then cp /mnt/wfs/inventory/vipscredentials.xml /mnt/wfs/sexigraf-dump/conf/vipscredentials.xml From 6f31d95f920b2c1b15e44410284638a5070fa5c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 11 Nov 2023 00:10:19 +0100 Subject: [PATCH 167/201] Update importSexiGrafBundle.sh #254 --- var/www/scripts/importSexiGrafBundle.sh | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/var/www/scripts/importSexiGrafBundle.sh b/var/www/scripts/importSexiGrafBundle.sh index 617e3818..fc28961a 100644 --- a/var/www/scripts/importSexiGrafBundle.sh +++ b/var/www/scripts/importSexiGrafBundle.sh @@ -38,20 +38,20 @@ if [ -d "/mnt/wfs/whisper" ]; then # chmod 644 /var/www/admin/offline-vminventory.html # Import credential store items - if [ -a "/var/www/.vmware/credstore/vicredentials.xml" ]; then - openssl des3 -d -salt -in /media/cdrom/conf/vicredentials.conf.ss -out /tmp/vicredentials.conf -pass pass:sexigraf -md md5 - # /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/CredstoreAdmin.ps1 -createstore -credstore /mnt/wfs/inventory/vipscredentials.xml - for creditem in $(cat /tmp/vicredentials.conf) - do - vcenter=$(echo $creditem | cut -d ";" -f 1) - username=$(echo $creditem | cut -d ";" -f 2) - password=$(echo $creditem | cut -d ";" -f 3) - # /usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml add --server $vcenter --username $username --password $password - /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/CredstoreAdmin.ps1 -credstore /mnt/wfs/inventory/vipscredentials.xml -add -server $vcenter -username $username -password $password - done - rm -f /tmp/vicredentials.conf - # chown www-data:www-data /mnt/wfs/inventory/vipscredentials.xml - fi + # if [ -a "/var/www/.vmware/credstore/vicredentials.xml" ]; then + # openssl des3 -d -salt -in /media/cdrom/conf/vicredentials.conf.ss -out /tmp/vicredentials.conf -pass pass:sexigraf -md md5 + # # /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/CredstoreAdmin.ps1 -createstore -credstore /mnt/wfs/inventory/vipscredentials.xml + # for creditem in $(cat /tmp/vicredentials.conf) + # do + # vcenter=$(echo $creditem | cut -d ";" -f 1) + # username=$(echo $creditem | cut -d ";" -f 2) + # password=$(echo $creditem | cut -d ";" -f 3) + # # /usr/lib/vmware-vcli/apps/general/credstore_admin.pl --credstore /var/www/.vmware/credstore/vicredentials.xml add --server $vcenter --username $username --password $password + # /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/CredstoreAdmin.ps1 -credstore /mnt/wfs/inventory/vipscredentials.xml -add -server $vcenter -username $username -password $password + # done + # rm -f /tmp/vicredentials.conf + # # chown www-data:www-data /mnt/wfs/inventory/vipscredentials.xml + # fi if [ -a "/media/cdrom/conf/vipscredentials.xml" ]; then /bin/cp -fR /media/cdrom/conf/vipscredentials.xml /mnt/wfs/inventory/vipscredentials.xml From 1d25c3367439c4e4292afb5d6f14e5482e23fcc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sat, 11 Nov 2023 00:39:05 +0100 Subject: [PATCH 168/201] Update VMware_VBR_Inventory.json --- .../dashboards/VMware_Inventory/VMware_VBR_Inventory.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json index 222c2c37..7969e1b9 100644 --- a/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json +++ b/var/lib/grafana/dashboards/VMware_Inventory/VMware_VBR_Inventory.json @@ -71,7 +71,8 @@ "style": "dark", "tags": [ "VI", - "VM" + "VM", + "VBR" ], "templating": { "list": [] @@ -85,6 +86,6 @@ }, "timezone": "", "title": "VMware VBR Inventory", - "uid": "iQlLNW57b", + "uid": "iQlLNW57c", "version": 8 } \ No newline at end of file From 3695a6c13cae7d96824175e56a45ebe9f20297e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 15 Nov 2023 10:34:03 +0100 Subject: [PATCH 169/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index ab8acd95..8e666c5d 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.39" +$ScriptVersion = "0.9.41" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -366,7 +366,7 @@ if ($($VbrJobsStates.data)) { $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ $vm_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name - if ($ViVmInventoryTable[$vm_name]) { + if ($ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name]) { $vcenter_name = NameCleaner $ViVmInventoryTable[$vm_name].vCenter if ($ViVmInventoryTable[$vm_name].Cluster) { $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].Cluster @@ -377,17 +377,17 @@ if ($($VbrJobsStates.data)) { $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount $VbrObjectInventoryInfo = "" | Select-Object VbrServer, JobName, vCenter, Cluster, VM, RestorePointsCount, LastRestorePoint - $VbrObjectInventoryInfo.VbrServer = $vbrserver_name - $VbrObjectInventoryInfo.JobName = $job_name - $VbrObjectInventoryInfo.vCenter = $vcenter_name - $VbrObjectInventoryInfo.Cluster = $cluster_name - $VbrObjectInventoryInfo.VM = $vm_name + $VbrObjectInventoryInfo.VbrServer = $Server + $VbrObjectInventoryInfo.JobName = $VbrSessions5[$VbrObjectRestorePoint.backupId].name + $VbrObjectInventoryInfo.vCenter = $ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name].vCenter + $VbrObjectInventoryInfo.Cluster = $ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name].Cluster + $VbrObjectInventoryInfo.VM = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name $VbrObjectInventoryInfo.RestorePointsCount = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount $VbrObjectInventoryInfo.LastRestorePoint = $VbrObjectRestorePoint.creationTime - if ($VbrVmInventoryTable[$vm_name]) { - $VbrVmInventoryTable[$vm_name] = $VbrObjectInventoryInfo + if ($VbrVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name]) { + $VbrVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name] = $VbrObjectInventoryInfo } else { - $VbrVmInventoryTable.add($vm_name,$VbrObjectInventoryInfo) + $VbrVmInventoryTable.add($VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name,$VbrObjectInventoryInfo) } } } @@ -418,7 +418,7 @@ if ($($VbrJobsStates.data)) { try { $VbrVmInventoryCsv = Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Sort-Object LastRestorePoint -Descending $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force - if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -eq "1") { $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -ErrorAction Stop -Force } } catch { From 09529e66be28cf154bbdbc426a4d73974cd29538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 15 Nov 2023 10:34:08 +0100 Subject: [PATCH 170/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 67277d5f..bda15076 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.85" +$ScriptVersion = "0.9.86" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -398,7 +398,7 @@ if ($ViServersList.count -gt 0) { # } Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory file ..." $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.csv -Force -ErrorAction Stop - if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -eq "1") { $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop } } catch { @@ -445,7 +445,7 @@ if ($ViServersList.count -gt 0) { # } Write-Host "$((Get-Date).ToString("o")) [INFO] Writing ESX Inventory file ..." $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force -ErrorAction Stop - if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -eq "1") { $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop } } catch { @@ -471,7 +471,7 @@ if ($ViServersList.count -gt 0) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Datastore Inventory files ..." $ViDatastoresInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViDsInventory.csv -Force -ErrorAction Stop - if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -match "1") { + if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -eq "1") { $ViDatastoresInfos|Out-File -Path /mnt/wfs/inventory/ViDsInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop } } catch { From 6f3e6e8c5769bcf18da3559426449262a2d3e495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 15 Nov 2023 10:34:11 +0100 Subject: [PATCH 171/201] Update VbrVmInventory.csv --- opt/sexigraf/VbrVmInventory.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opt/sexigraf/VbrVmInventory.csv b/opt/sexigraf/VbrVmInventory.csv index b6c16e1d..01ce379d 100644 --- a/opt/sexigraf/VbrVmInventory.csv +++ b/opt/sexigraf/VbrVmInventory.csv @@ -1 +1 @@ -"VbrServer", "JobName", "vCenter", "Cluster", "VM", "RestorePointsCount", "LastRestorePoint" \ No newline at end of file +"VbrServer","JobName","vCenter","Cluster","VM","RestorePointsCount","LastRestorePoint" \ No newline at end of file From 7c79892c505b123249848bf3a710348eabb47a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Mon, 20 Nov 2023 18:41:35 +0100 Subject: [PATCH 172/201] Update ViPullStatistics.ps1 --- opt/sexigraf/ViPullStatistics.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 26d93176..7da93229 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([parameter (Mandatory=$true)] [string] $Server, [parameter (Mandatory=$true)] [string] $SessionFile, [parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1032" +$ScriptVersion = "0.9.1033" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -290,6 +290,9 @@ try { SexiLogger "[INFO] Sessions.TerminateSession privilege not detected, SessionManager skipped ..." } $EventManager = Get-View $ServiceInstance.Content.EventManager -Property latestEvent, description -Server $Server + if ($vSanPull = Test-Path -Path $("/etc/cron.d/vsan_" + $Server.Replace(".","_"))) { + SexiLogger "[INFO] vSAN collector enabled ..." + } } catch { AltAndCatchFire "AuthorizationManager or SessionManager check failure" } From 42f54a5d5cbd9a5edc6f3337e8829941ff66ac31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 28 Nov 2023 22:21:37 +0100 Subject: [PATCH 173/201] Create VMware_Evo_Version.json #365 --- .../VMware_vCenter/VMware_Evo_Version.json | 469 ++++++++++++++++++ 1 file changed, 469 insertions(+) create mode 100644 var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json diff --git a/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json b/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json new file mode 100644 index 00000000..9724030e --- /dev/null +++ b/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json @@ -0,0 +1,469 @@ +{ + "annotations": { + "list": [ + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "iteration": 1617871211625, + "links": [], + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 61, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": null, + "hide": false, + "refId": "A", + "target": "groupByNode(vi.$vcenter.vi.version.vm.$datacentre.$cluster.vmtools.*, 8, 'sum')" + } + ], + "title": "VMTools version", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 61, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": null, + "hide": false, + "refId": "A", + "target": "groupByNode(vi.$vcenter.vi.version.vm.$datacentre.$cluster.guest.*, 8, 'sum')" + } + ], + "title": "GuestId version", + "type": "timeseries" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 61, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": null, + "hide": false, + "refId": "A", + "target": "groupByNode(vi.$vcenter.vi.version.vm.$datacentre.$cluster.vhw.*, 8, 'sum')" + } + ], + "title": "vHardware version", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.4", + "targets": [ + { + "datasource": null, + "hide": false, + "refCount": 0, + "refId": "B", + "target": "groupByNode(removeEmptySeries(vi.$vcenter.vi.version.esx.$datacentre.$cluster.build.*), 8, 'sum')" + } + ], + "title": "ESX version", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 27, + "style": "light", + "tags": [ + "VI" + ], + "templating": { + "list": [ + { + "allFormat": "wildcard", + "allValue": null, + "current": {}, + "datasource": null, + "definition": "vi.*", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "vcenter", + "options": [], + "query": "vi.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": null, + "definition": "vi.$vcenter.vi.version.esx.*", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "datacentre", + "options": [], + "query": "vi.$vcenter.vi.version.esx.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allFormat": "wildcard", + "allValue": "*", + "current": {}, + "datasource": null, + "definition": "vi.$vcenter.vi.version.esx.$datacentre.*", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "multiFormat": "glob", + "name": "cluster", + "options": [], + "query": "vi.$vcenter.vi.version.esx.$datacentre.*", + "refresh": 1, + "refresh_on_load": true, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1d", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "1d", + "2d", + "3d", + "4d", + "5d", + "6d", + "7d", + "15d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "VMware Evo Version", + "uid": "v5Re8clMz", + "version": 30 +} \ No newline at end of file From 8614c87d6b658b1e183ef3f3cf0af0cb66adcd45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 28 Nov 2023 22:58:07 +0100 Subject: [PATCH 174/201] Update VMware_Evo_Version.json --- .../grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json b/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json index 9724030e..838af96c 100644 --- a/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json +++ b/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json @@ -306,7 +306,7 @@ "overrides": [] }, "gridPos": { - "h": 15, + "h": 14, "w": 24, "x": 0, "y": 42 From 5e066caa988abe2f91212101d3056240247f6c41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 28 Nov 2023 22:58:12 +0100 Subject: [PATCH 175/201] #275 --- opt/sexigraf/ViOfflineInventory.ps1 | 10 +++++----- opt/veeam/VbrPullStatistics.ps1 | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index bda15076..4a06575e 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.86" +$ScriptVersion = "0.9.87" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -364,10 +364,10 @@ if ($ViServersList.count -gt 0) { if ($(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv")) { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViVmInventory.*.csv files ..." - $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} + $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { - Get-ChildItem $ExtraCsvFiles.name | Remove-Item -Force -Confirm:$false + Get-ChildItem "/mnt/wfs/inventory/$($ExtraCsvFiles.name)" | Remove-Item -Force -Confirm:$false } catch { AltAndCatchFire "Cannot remove extra csv files" } @@ -411,7 +411,7 @@ if ($ViServersList.count -gt 0) { if ($(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv")) { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViEsxInventory.*.csv files ..." - $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} + $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { Get-ChildItem $ExtraCsvFiles.name | Remove-Item -Force -Confirm:$false @@ -458,7 +458,7 @@ if ($ViServersList.count -gt 0) { if ($(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv")) { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViDsInventory.*.csv files ..." - $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} + $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { Get-ChildItem $ExtraCsvFiles.name | Remove-Item -Force -Confirm:$false diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index 8e666c5d..f335e254 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.41" +$ScriptVersion = "0.9.42" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -404,7 +404,7 @@ if ($($VbrJobsStates.data)) { if ($(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv")) { SexiLogger "[INFO] Rotating VbrVmInventory.*.csv files ..." - $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property Name | ?{$_.SideIndicator -eq "<="} + $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { Get-ChildItem $ExtraCsvFiles.name | Remove-Item -Force -Confirm:$false From 83f2da569f30a0e58d41aa6bf41c4d6009adeed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 29 Nov 2023 16:28:31 +0100 Subject: [PATCH 176/201] Update VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 index f335e254..2cbaaaf5 100644 --- a/opt/veeam/VbrPullStatistics.ps1 +++ b/opt/veeam/VbrPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.42" +$ScriptVersion = "0.9.43" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -407,7 +407,7 @@ if ($($VbrJobsStates.data)) { $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { - Get-ChildItem $ExtraCsvFiles.name | Remove-Item -Force -Confirm:$false + Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false } catch { AltAndCatchFire "Cannot remove extra csv files" } From 411e8bac910c5b68f3b0ba0f2d0f4768f54b0a2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 29 Nov 2023 16:28:33 +0100 Subject: [PATCH 177/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index 4a06575e..eee4e7c8 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.87" +$ScriptVersion = "0.9.88" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -367,7 +367,7 @@ if ($ViServersList.count -gt 0) { $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { - Get-ChildItem "/mnt/wfs/inventory/$($ExtraCsvFiles.name)" | Remove-Item -Force -Confirm:$false + Get-ChildItem "/mnt/wfs/inventory/$($ExtraCsvFiles.FullName)" | Remove-Item -Force -Confirm:$false } catch { AltAndCatchFire "Cannot remove extra csv files" } @@ -414,7 +414,7 @@ if ($ViServersList.count -gt 0) { $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { - Get-ChildItem $ExtraCsvFiles.name | Remove-Item -Force -Confirm:$false + Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false } catch { AltAndCatchFire "Cannot remove extra csv files" } @@ -461,7 +461,7 @@ if ($ViServersList.count -gt 0) { $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { - Get-ChildItem $ExtraCsvFiles.name | Remove-Item -Force -Confirm:$false + Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false } catch { AltAndCatchFire "Cannot remove extra csv files" } From 3bc8d9d1430230912e29bd97792b7fcc21a1a90f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 1 Dec 2023 00:05:18 +0100 Subject: [PATCH 178/201] Update storage-aggregation.conf --- opt/graphite/conf/storage-aggregation.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opt/graphite/conf/storage-aggregation.conf b/opt/graphite/conf/storage-aggregation.conf index b2510da1..affb3ee0 100644 --- a/opt/graphite/conf/storage-aggregation.conf +++ b/opt/graphite/conf/storage-aggregation.conf @@ -71,4 +71,4 @@ aggregationMethod = last [veeamStatus] pattern = ^veeam.*\.status xFilesFactor = 0.0 -aggregationMethod = last \ No newline at end of file +aggregationMethod = last From 408b5714ab9cb29fdc3e6a8f7db939f1b557c6a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 1 Dec 2023 00:05:20 +0100 Subject: [PATCH 179/201] Update storage-schemas.conf --- opt/graphite/conf/storage-schemas.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opt/graphite/conf/storage-schemas.conf b/opt/graphite/conf/storage-schemas.conf index ed2f8d00..e113c66a 100644 --- a/opt/graphite/conf/storage-schemas.conf +++ b/opt/graphite/conf/storage-schemas.conf @@ -58,4 +58,4 @@ retentions = 5m:7d,10m:30d,15m:90d,30m:120d,120m:1y [anything_else] pattern = .* -retentions = 5m:24h,10m:48h,30m:96h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y \ No newline at end of file +retentions = 5m:24h,10m:48h,30m:96h,60m:7d,240m:30d,720m:90d,2880m:1y,5760m:2y,17280m:5y From c4ae3f4121b9d5d988293fd830526351fa86d313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Fri, 1 Dec 2023 00:41:09 +0100 Subject: [PATCH 180/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index eee4e7c8..e648830a 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.88" +$ScriptVersion = "0.9.89" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -367,7 +367,7 @@ if ($ViServersList.count -gt 0) { $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { - Get-ChildItem "/mnt/wfs/inventory/$($ExtraCsvFiles.FullName)" | Remove-Item -Force -Confirm:$false + Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false } catch { AltAndCatchFire "Cannot remove extra csv files" } From 5d01144d2f1f7731c8402b8eb365c07f22594e3d Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 1 Dec 2023 12:08:03 +0100 Subject: [PATCH 181/201] Update VMware_All_Version.json --- .../grafana/dashboards/VMware_vCenter/VMware_All_Version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json b/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json index 1ebabb24..98515710 100644 --- a/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json +++ b/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json @@ -504,7 +504,7 @@ "list": [ { "allFormat": "wildcard", - "allValue": null, + "allValue": "*", "current": {}, "datasource": null, "definition": "vi.*", From 3691dab7efa551832eaa7b3234eb64ef6e95d534 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Sat, 2 Dec 2023 10:08:02 +0100 Subject: [PATCH 182/201] Update VMware_All_Version.json --- .../grafana/dashboards/VMware_vCenter/VMware_All_Version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json b/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json index 98515710..f2140d8a 100644 --- a/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json +++ b/var/lib/grafana/dashboards/VMware_vCenter/VMware_All_Version.json @@ -511,7 +511,7 @@ "description": null, "error": null, "hide": 0, - "includeAll": false, + "includeAll": true, "label": null, "multi": true, "multiFormat": "glob", From 4fdabf5d34edc522f31d8ebbb0703a05a690d010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 5 Dec 2023 23:44:01 +0100 Subject: [PATCH 183/201] Delete VbrPullStatistics.ps1 --- opt/veeam/VbrPullStatistics.ps1 | 475 -------------------------------- 1 file changed, 475 deletions(-) delete mode 100644 opt/veeam/VbrPullStatistics.ps1 diff --git a/opt/veeam/VbrPullStatistics.ps1 b/opt/veeam/VbrPullStatistics.ps1 deleted file mode 100644 index 2cbaaaf5..00000000 --- a/opt/veeam/VbrPullStatistics.ps1 +++ /dev/null @@ -1,475 +0,0 @@ -#!/usr/bin/pwsh -Command -# -param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) - -$ScriptVersion = "0.9.43" - -$ExecStart = $(Get-Date).ToUniversalTime() -# $stopwatch = [system.diagnostics.stopwatch]::StartNew() - -$ErrorActionPreference = "SilentlyContinue" -$WarningPreference = "SilentlyContinue" -(Get-Process -Id $pid).PriorityClass = 'Idle' -$SexiVbrMutex = New-Object System.Threading.Mutex($false, "SexiVbrMutex") -# https://learn-powershell.net/2014/09/30/using-mutexes-to-write-data-to-the-same-logfile-across-processes-with-powershell/ -# https://github.com/ryan-leap/GreenMeansGoMutexDemo - -function SexiLogger { - param($Text2Log) - Write-Host "$((Get-Date).ToString("o")) $Text2Log" - $null = $SexiVbrMutex.WaitOne(500) - Add-Content -Path "/var/log/sexigraf/VbrPullStatistics.log" -Value "$((Get-Date).ToString("o")) $($Server) $Text2Log" - $SexiVbrMutex.ReleaseMutex() -} - -function AltAndCatchFire { - Param($ExitReason) - SexiLogger "[EROR] $ExitReason" - SexiLogger "[EROR] $($Error[0])" - SexiLogger "[EROR] Exit" - Stop-Transcript - exit -} - -function NameCleaner { - Param($NameToClean) - $NameToClean = $NameToClean -replace "[ .]","_" - [System.Text.NormalizationForm]$NormalizationForm = "FormD" - $NameToClean = $NameToClean.Normalize($NormalizationForm) - $NameToClean = $NameToClean -replace "[^[:ascii:]]","" -replace "[^A-Za-z0-9-_]","_" - return $NameToClean.ToLower() -} - - -try { - Start-Transcript -Path "/var/log/sexigraf/VbrPullStatistics.$($Server).log" -Append -Confirm:$false -Force -UseMinimalHeader - SexiLogger "[INFO] VbrPullStatistics v$ScriptVersion" -} catch { - SexiLogger "[EROR] VbrPullStatistics logging failure" - SexiLogger "[EROR] Exit" - exit -} - -try { - SexiLogger "[INFO] Importing Graphite PowerShell module ..." - Import-Module -Name /usr/local/share/powershell/Modules/Graphite-PowerShell-Functions/Graphite-Powershell.psm1 -Global -Force -SkipEditionCheck -} catch { - AltAndCatchFire "Powershell modules import failure" -} - -try { - SexiLogger "[INFO] Looking for another VbrPullStatistics for $Server ..." - $DupVbrPullStatisticsProcess = Get-PSHostProcessInfo|%{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}|?{$_ -match "VbrPullStatistics" -and $_ -match "$Server"} - # https://github.com/PowerShell/PowerShell/issues/13944 - if (($DupVbrPullStatisticsProcess|Measure-Object).Count -gt 1) { - $DupVbrPullStatisticsProcessId = (Get-PSHostProcessInfo|?{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '|?{$_ -match "$Server"}}).ProcessId[0] - $DupVbrPullStatisticsProcessTime = [INT32](ps -p $DupVbrPullStatisticsProcessId -o etimes).split()[-1] - if ($DupVbrPullStatisticsProcessTime -gt 300) { - SexiLogger "[WARN] VbrPullStatistics for $Server is already running for more than 5 minutes!" - SexiLogger "[WARN] Killing stunned VbrPullStatistics for $Server" - Stop-Process -Id $DupVbrPullStatisticsProcessId -Force - } else { - AltAndCatchFire "VbrPullStatistics for $Server is already running!" - } - } -} catch { - AltAndCatchFire "VbrPullStatistics process lookup failure" -} - -if ($SessionFile) { - try { - $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" - if ([DateTime]$(Get-Content -Path /tmp/$SessionSecretExpiration) -gt $ExecStart.AddMinutes(5)) { - $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop - SexiLogger "[INFO] SessionToken found in SessionFile, attempting connection to $Server ..." - $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} - $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders - if ($($VbrJobsStates.data)) { - SexiLogger "[INFO] Connected to VBR REST API Server $Server" - } else { - SexiLogger "[WARN] Connection failure or no job state" - } - } else { - SexiLogger "[WARN] Token has expired or is about to ..." - } - } catch { - SexiLogger "[WARN] SessionToken or SessionSecretExpiration not found, invalid or connection failure" - } - - if (!$($VbrJobsStates.data)) { - SexiLogger "[WARN] Attempting token refresh ..." - try { - $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" - $SessionRefresh = Get-Content -Path /tmp/$SessionRefreshPath -ErrorAction Stop - $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} - $VbrBody = @{grant_type = "refresh_token";username = "";password = "";refresh_token = $SessionRefresh;code = "";use_short_term_refresh = ""} - $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody - if ($VbrConnect.access_token) { - $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" - $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" - $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName - $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefreshPath - $VbrConnect.".expires".ToUniversalTime().tostring() | Out-File -FilePath /tmp/$SessionSecretExpiration - $SessionToken = $VbrConnect.access_token - $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} - $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders - if (!$($VbrJobsStates.data)) { - SexiLogger "[WARN] Token refresh failed!" - SexiLogger "[WARN] Known issue on v11!" - # https://forums.veeam.com/restful-api-f30/how-to-handle-the-refresh-token-t74916.html - } - } else { - SexiLogger "[WARN] Token refresh failed!" - } - } catch { - SexiLogger "[WARN] Token refresh issue!" - } - } - - if (!$($VbrJobsStates.data)) { - SexiLogger "[WARN] Attempting explicit connection ..." - try { - $createstorexml = New-Object -TypeName XML - $createstorexml.Load($credstore) - $XPath = '//passwordEntry[server="' + $Server + '"]' - if ($(Select-XML -Xml $createstorexml -XPath $XPath)){ - $item = Select-XML -Xml $createstorexml -XPath $XPath - $CredStoreLogin = $item.Node.username - $CredStorePassword = [System.Text.Encoding]::Unicode.GetString([System.Convert]::FromBase64String($item.Node.password)) - } else { - AltAndCatchFire "No $Server entry in CredStore" - } - $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} - $VbrBody = @{grant_type = "password";username = $CredStoreLogin;password = $CredStorePassword;refresh_token = "";code = "";use_short_term_refresh = ""} - $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody - if ($VbrConnect.access_token) { - $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} - $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders - if ($($VbrJobsStates.data)) { - SexiLogger "[INFO] Connected to VBR REST API Server $Server" - $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" - $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" - $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" - $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName - $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefreshPath - $VbrConnect.".expires".ToUniversalTime().tostring() | Out-File -FilePath /tmp/$SessionSecretExpiration - $SessionToken = $VbrConnect.access_token - } else { - AltAndCatchFire "Jobs States check failed, no job state or check the user permissions!" - } - } else { - AltAndCatchFire "Explicit connection failed, check the stored credentials!" - } - } catch { - AltAndCatchFire "Explicit connection failed, check the stored credentials!" - } - } -} else { - AltAndCatchFire "No SessionFile somehow ..." -} - -if ($($VbrJobsStates.data)) { - $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} - $VbrDataTable = @{} - $vbrserver_name = NameCleaner $Server - SexiLogger "[INFO] Start processing VBR Server $Server ..." - - $VbrJobsStatesTable = @{} - foreach ($VbrJobState in $VbrJobsStates.data) { - try { - $VbrJobsStatesTable.add($VbrJobState.id,$VbrJobState) - } catch {} - $job_name = NameCleaner $VbrJobState.name - if ($VbrJobState.status -eq "running") { - $VbrJobStateStatus = 0 - } elseif ($VbrJobState.status -eq "inactive") { - $VbrJobStateStatus = 1 - } elseif ($VbrJobState.status -eq "disabled") { - $VbrJobStateStatus = 2 - } else { - $VbrJobStateStatus = 3 - } - $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.status"] = $VbrJobStateStatus - } - - try { - SexiLogger "[INFO] VBR repositories states collect ..." - $VbrRepositoriesStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupInfrastructure/repositories/states") -Headers $VbrAuthHeaders - } catch { - SexiLogger "[EROR] repositories states collect failure" - SexiLogger "[EROR] $($Error[0])" - } - - if ($VbrRepositoriesStates.data) { - SexiLogger "[INFO] VBR repositories processing ..." - $VbrRepositoryTable = @{} - foreach ($VbrRepository in $VbrRepositoriesStates.data) { - $VbrRepositoryName = NameCleaner $VbrRepository.name - - $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.capacityGB"] = $VbrRepository.capacityGB - $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.freeGB"] = $VbrRepository.freeGB - $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.usedSpaceGB"] = $VbrRepository.usedSpaceGB - $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.RealUsedPct"] = $($VbrRepository.usedSpaceGB * 100 / $($VbrRepository.usedSpaceGB + $VbrRepository.freeGB )) - - try { - $VbrRepositoryTable.add($VbrRepository.id,$VbrRepository) - } catch {} - } - } else { - SexiLogger "[WARN] No repositories ?!" - } - - try { - SexiLogger "[INFO] VBR ScaleOutRepositories collect ..." - $VbrScaleOutRepositories = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupInfrastructure/scaleOutRepositories") -Headers $VbrAuthHeaders - } catch { - SexiLogger "[EROR] ScaleOutRepositories collect failure" - SexiLogger "[EROR] $($Error[0])" - } - - if ($VbrScaleOutRepositories.data) { - SexiLogger "[INFO] SOBR performanceTier processing ..." - foreach ($VbrScaleOutRepository in $VbrScaleOutRepositories.data) { - try { - $VbrScaleOutRepositoryName = NameCleaner $VbrScaleOutRepository.name - - $VbrSobrNotNormal = $VbrScaleOutRepository.performanceTier.performanceExtents|?{$_.status -ne "Normal"} - - if ($VbrSobrNotNormal.id) { - $VbrSobrNotNormalRepos = $VbrRepositoryTable[$VbrSobrNotNormal.id] - $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.capacityGB"] += $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property capacityGB).Sum - $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] += $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property usedSpaceGB).Sum - } - - $VbrSobrNormal = $VbrScaleOutRepository.performanceTier.performanceExtents|?{$_.status -eq "Normal"} - if ($VbrSobrNormal.id) { - $VbrSobrNormalRepos = $VbrRepositoryTable[$VbrSobrNormal.id] - $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.capacityGB"] += $($VbrSobrNormalRepos|Measure-Object -Sum -Property capacityGB).Sum - $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] += $($VbrSobrNormalRepos|Measure-Object -Sum -Property usedSpaceGB).Sum - $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"] = $($VbrSobrNormalRepos|Measure-Object -Sum -Property freeGB).Sum - $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.RealUsedPct"] = $($VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] * 100 / $($VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"])) - } else { - $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"] = 0 - $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.RealUsedPct"] = 100 - } - } catch { - SexiLogger "[EROR] Issue processing SOBR $($VbrScaleOutRepository.name)" - SexiLogger "[EROR] $($Error[0])" - } - } - } else { - SexiLogger "[INFO] No SOBR" - } - - try { - SexiLogger "[INFO] VBR 5min old objectRestorePoints collect ..." - # $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders - $VbrObjectRestorePoints5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/objectRestorePoints?platformNameFilter=VmWare&limit=999&createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders - } catch { - SexiLogger "[EROR] objectRestorePoints collect failure" - SexiLogger "[EROR] $($Error[0])" - } - - # $VbrVmwareServers = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/inventory/vmware/hosts") -Headers $VbrAuthHeaders - - - if ($VbrObjectRestorePoints5.data) { - # SexiLogger "[INFO] VBR backupObjects collect ..." - # $VbrBackupObjects5 = @{} - # foreach ($VbrObjectRestorePoint5 in $VbrObjectRestorePoints5.data) { - # try { - # $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&nameFilter=" + $($VbrObjectRestorePoint5.name)) -Headers $VbrAuthHeaders - # if ($VbrBackupObjects.data[0].type -eq "VM" -and !$VbrBackupObjects5[$VbrBackupObjects.data[0].name]) { - # $VbrBackupObjects5.Add($VbrBackupObjects.data[0].name,$VbrBackupObjects[0].data) - # } - # } catch { - # SexiLogger "[EROR] backupObjects collect failure" - # SexiLogger "[EROR] $($Error[0])" - # } - # } # Too Slow !!! - - try { - SexiLogger "[INFO] VBR backupObjects collect ..." - $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&limit=9999") -Headers $VbrAuthHeaders # TODO limit=9999+ ? - if ($VbrBackupObjects.data) { - SexiLogger "[INFO] Building VBR backupObjects table ..." - $VbrBackupObjectsTable = @{} - foreach ($VbrBackupObject in $VbrBackupObjects.data) { - try { - $VbrBackupObjectsTable.add($VbrBackupObject.name,$VbrBackupObject) - } catch {} - } - } - } catch { - SexiLogger "[EROR] backupObjects collect failure" - SexiLogger "[EROR] $($Error[0])" - } - - $VbrObjectRestorePoints5SessionsId = $VbrObjectRestorePoints5.data.backupId|Select-Object -Unique - - SexiLogger "[INFO] VBR Sessions collect ..." - $VbrSessions5 = @{} - foreach ($VbrObjectRestorePoints5SessionId in $VbrObjectRestorePoints5SessionsId) { - try { - $VbrObjectRestorePoints5Session = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backups/" + $VbrObjectRestorePoints5SessionId) -Headers $VbrAuthHeaders - if ($VbrObjectRestorePoints5Session|?{$_.jobId -ne "00000000-0000-0000-0000-000000000000"}) { - $VbrSessions5.Add($VbrObjectRestorePoints5SessionId,$VbrObjectRestorePoints5Session) - } - } catch { - SexiLogger "[EROR] backupObjects collect failure" - SexiLogger "[EROR] $($Error[0])" - } - } - - if (Test-Path /mnt/wfs/inventory/ViVmInventory.csv) { - SexiLogger "[INFO] Importing VM inventory ..." - try { - $ViVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop) - } catch { - SexiLogger "[EROR] ViVmInventory import issue" - SexiLogger "[EROR] $($Error[0])" - } - - if ($ViVmInventory) { - $ViVmInventoryTable = @{} - foreach ($ViVm in $ViVmInventory) { - try { - $ViVmInventoryTable.Add($ViVm.vm,$ViVm) - } catch {} - } - - if (Test-Path /mnt/wfs/inventory/VbrInv_$($Server).csv) { - SexiLogger "[INFO] Importing $($Server) VBR inventory ..." - try { - $VbrVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/VbrInv_$($Server).csv -ErrorAction Stop) - } catch { - SexiLogger "[EROR] VbrVmInventory import issue" - SexiLogger "[EROR] $($Error[0])" - } - if ($ViVmInventory) { - $VbrVmInventoryTable = @{} - foreach ($VbrVm in $VbrVmInventory) { - try { - if ($ViVmInventoryTable[$VbrVm.vm]) { - $VbrVmInventoryTable.Add($VbrVm.vm,$VbrVm) - } - } catch {} - } - } - } else { - $VbrVmInventoryTable = @{} - } - - SexiLogger "[INFO] Building $($Server) VBR DataTable and inventory ..." - foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { - if ($VbrJobsStatesTable[$VbrSessions5[$VbrObjectRestorePoint.backupId].jobId]) { - $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name - $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ - $vm_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name - if ($ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name]) { - $vcenter_name = NameCleaner $ViVmInventoryTable[$vm_name].vCenter - if ($ViVmInventoryTable[$vm_name].Cluster) { - $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].Cluster - } else { - $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].ESX - } - $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.objectRestorePoints"] ++ - $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount - - $VbrObjectInventoryInfo = "" | Select-Object VbrServer, JobName, vCenter, Cluster, VM, RestorePointsCount, LastRestorePoint - $VbrObjectInventoryInfo.VbrServer = $Server - $VbrObjectInventoryInfo.JobName = $VbrSessions5[$VbrObjectRestorePoint.backupId].name - $VbrObjectInventoryInfo.vCenter = $ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name].vCenter - $VbrObjectInventoryInfo.Cluster = $ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name].Cluster - $VbrObjectInventoryInfo.VM = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name - $VbrObjectInventoryInfo.RestorePointsCount = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount - $VbrObjectInventoryInfo.LastRestorePoint = $VbrObjectRestorePoint.creationTime - if ($VbrVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name]) { - $VbrVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name] = $VbrObjectInventoryInfo - } else { - $VbrVmInventoryTable.add($VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name,$VbrObjectInventoryInfo) - } - } - } - } - - if ($VbrVmInventoryTable.Values) { - SexiLogger "[INFO] Exporting $($Server) VBR inventory ..." - try { - $VbrVmInventoryTable.Values|Sort-Object VM|Export-Csv /mnt/wfs/inventory/VbrInv_$($Server).csv -ErrorAction Stop -Force - } catch { - SexiLogger "[EROR] VbrVmInventory import issue" - SexiLogger "[EROR] $($Error[0])" - } - - if ($(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv")) { - SexiLogger "[INFO] Rotating VbrVmInventory.*.csv files ..." - $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} - If ($ExtraCsvFiles) { - try { - Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false - } catch { - AltAndCatchFire "Cannot remove extra csv files" - } - } - } - - SexiLogger "[INFO] Import/Export VBR inventories ..." - try { - $VbrVmInventoryCsv = Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Sort-Object LastRestorePoint -Descending - $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force - if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -eq "1") { - $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -ErrorAction Stop -Force - } - } catch { - SexiLogger "[EROR] Import/Export VBR inventories issue" - SexiLogger "[EROR] $($Error[0])" - } - } else { - SexiLogger "[EROR] No VbrVmInventoryTable values ?!" - } - } else { - SexiLogger "[EROR] ViVmInventory import issue" - } - } else { - SexiLogger "[EROR] No ViVmInventory" - } - } - - try { - SexiLogger "[INFO] VBR ended sessions collect ..." - $VbrEndedSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?endedAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders - } catch { - SexiLogger "[EROR] Sessions collect failure" - SexiLogger "[EROR] $($Error[0])" - } - - if ($VbrEndedSessions.data) { - SexiLogger "[INFO] Processing VBR ended sessions ..." - foreach ($VbrEndedSession in $VbrEndedSessions.data) { - $job_name = NameCleaner $VbrEndedSession.name - if ($VbrEndedSession.result.result -eq "Success") { - $VbrEndedSessionResult = 0 - } elseif ($VbrEndedSession.result.result -eq "Warning") { - $VbrEndedSessionResult = 1 - } elseif ($VbrEndedSession.result.result -eq "Failed") { - $VbrEndedSessionResult = 2 - } else { - $VbrEndedSessionResult = 3 - } - $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.result"] = $VbrEndedSessionResult - } - } - - $VbrDataTable["veeam.vbr.$vbrserver_name.exec.duration"] = $($(Get-Date).ToUniversalTime() - $ExecStart).TotalSeconds - SexiLogger "[INFO] Sending veeam data to Graphite for VBR server $Server ..." - Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrDataTable -DateTime $ExecStart - - SexiLogger "[INFO] End of VBR server $server processing ..." - -} else { - SexiLogger "[INFO] No Running Jobs on VBR server $server ..." - SexiLogger "[INFO] Exit" - Stop-Transcript - exit -} \ No newline at end of file From 0d390214b76b11720f9202b93219307edac9cbb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 5 Dec 2023 23:44:03 +0100 Subject: [PATCH 184/201] Delete VbrConnect.ps1 --- opt/veeam/VbrConnect.ps1 | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 opt/veeam/VbrConnect.ps1 diff --git a/opt/veeam/VbrConnect.ps1 b/opt/veeam/VbrConnect.ps1 deleted file mode 100644 index c5a7baa6..00000000 --- a/opt/veeam/VbrConnect.ps1 +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/pwsh -NonInteractive -NoProfile -Command -# -param([Parameter (Mandatory=$true)] [string] $server, [Parameter (Mandatory=$true)] [string] $username, [Parameter (Mandatory=$true)] [string] $password) - -# https://communities.vmware.com/t5/VMware-PowerCLI-Discussions/PowerCLI-on-Debian-Stretch-The-type-initializer-for-VMware/m-p/451739#M10646 -Set-Content -Path Env:HOME -Value '/tmp' - -if (!$(Test-Connection -TargetName $server -TcpPort 9419 -TimeoutSeconds 2)) { - Write-Host "$server is not answering at TCP:9419 check if VeeamBackupRESTSvc is running" - exit 1 -} - -try { - $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} - $VbrBody = @{grant_type = "password";username = $username;password = $password;refresh_token = "";code = "";use_short_term_refresh = ""} - $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody - if ($VbrConnect.access_token) { - $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" - $SessionRefresh = "vbr_" + $server.Replace(".","_") + ".dat" - $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName - $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefresh - Write-Host "Connected to $server" - } else { - Write-Host "Connection to $server failed!" - exit 1 - } -} catch { - # Invoke-RestMethod: Unable to read data from the transport connection: Connection reset by peer. - # https://helpcenter.veeam.com/docs/backup/vbr_rest/tls_certificate.html - Write-Host "$($Error[0])" - exit 1 -} \ No newline at end of file From 7ea7b52220dfab635c5b9cb6fc67bd40dd536076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 5 Dec 2023 23:44:07 +0100 Subject: [PATCH 185/201] Update addVbrCrontab.sh --- var/www/scripts/addVbrCrontab.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/scripts/addVbrCrontab.sh b/var/www/scripts/addVbrCrontab.sh index 73528525..3fa1367b 100644 --- a/var/www/scripts/addVbrCrontab.sh +++ b/var/www/scripts/addVbrCrontab.sh @@ -1,5 +1,5 @@ #!/bin/bash sessionFile="/tmp/vbr_$(sed s/\\./_/g <<<$1).key" crontabFile="/etc/cron.d/vbr_$(sed s/\\./_/g <<<$1)" -echo "*/5 * * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/veeam/VbrPullStatistics.ps1 -credstore /mnt/wfs/inventory/vbrpscredentials.xml -server $1 -sessionfile $sessionFile >/dev/null 2>&1" >> $crontabFile +echo "*/5 * * * * root /usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/VbrPullStatistics.ps1 -credstore /mnt/wfs/inventory/vbrpscredentials.xml -server $1 -sessionfile $sessionFile >/dev/null 2>&1" >> $crontabFile service cron reload \ No newline at end of file From cdfe72dd597fc7806c29951258d23b52b6ae38bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 5 Dec 2023 23:44:08 +0100 Subject: [PATCH 186/201] Update vbrcredstore.php --- var/www/admin/vbrcredstore.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/admin/vbrcredstore.php b/var/www/admin/vbrcredstore.php index eb107f12..e7074032 100644 --- a/var/www/admin/vbrcredstore.php +++ b/var/www/admin/vbrcredstore.php @@ -93,7 +93,7 @@ $errorHappened = true; $errorMessage = "Wrong username format, supported format are DOMAIN\USERNAME, USERNAME, USERNAME@DOMAIN.TLD"; } else { - exec("/usr/bin/pwsh -NonInteractive -NoProfile -f /opt/veeam/VbrConnect.ps1 -server " . escapeshellcmd($_POST["input-vbr"]) . " -username " . escapeshellcmd($_POST["input-username"]) . " -password " . escapeshellcmd($_POST["input-password"]), $null, $return_var); + exec("/usr/bin/pwsh -NonInteractive -NoProfile -f /opt/sexigraf/VbrConnect.ps1 -server " . escapeshellcmd($_POST["input-vbr"]) . " -username " . escapeshellcmd($_POST["input-username"]) . " -password " . escapeshellcmd($_POST["input-password"]), $null, $return_var); if ($return_var) { $errorHappened = true; $errorMessage = "Wrong username/password, AccessDenied or no answer at TCP:9419"; From 11fa887d9b37640889fc5f861d221f051c173936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 5 Dec 2023 23:44:11 +0100 Subject: [PATCH 187/201] Update ViOfflineInventory.ps1 --- opt/sexigraf/ViOfflineInventory.ps1 | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/opt/sexigraf/ViOfflineInventory.ps1 b/opt/sexigraf/ViOfflineInventory.ps1 index e648830a..2f4bf464 100644 --- a/opt/sexigraf/ViOfflineInventory.ps1 +++ b/opt/sexigraf/ViOfflineInventory.ps1 @@ -3,7 +3,7 @@ param([Parameter (Mandatory=$true)] [string] $CredStore) -$ScriptVersion = "0.9.89" +$ScriptVersion = "0.9.90" $ErrorActionPreference = "SilentlyContinue" $WarningPreference = "SilentlyContinue" @@ -362,9 +362,11 @@ if ($ViServersList.count -gt 0) { if ($ViVmsInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv")) { + $ViVmInventories = Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv" + + if ($ViVmInventories) { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViVmInventory.*.csv files ..." - $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViVmInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} + $ExtraCsvFiles = Compare-Object $ViVmInventories $($ViVmInventories|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false @@ -398,7 +400,7 @@ if ($ViServersList.count -gt 0) { # } Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Vm Inventory file ..." $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.csv -Force -ErrorAction Stop - if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -eq "1") { + if ($ExecStart.DayOfWeek -match "Monday" -and !$($ViVmInventories|?{$_.LastWriteTime -gt $ExecStart.AddDays(-1)})) { $ViVmsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop } } catch { @@ -409,9 +411,11 @@ if ($ViServersList.count -gt 0) { if ($ViEsxsInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv")) { + $ViEsxInventories = Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv" + + if ($ViEsxInventories) { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViEsxInventory.*.csv files ..." - $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViEsxInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} + $ExtraCsvFiles = Compare-Object $ViEsxInventories $($ViEsxInventories|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false @@ -445,7 +449,7 @@ if ($ViServersList.count -gt 0) { # } Write-Host "$((Get-Date).ToString("o")) [INFO] Writing ESX Inventory file ..." $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.csv -Force -ErrorAction Stop - if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -eq "1") { + if ($ExecStart.DayOfWeek -match "Monday" -and !$($ViEsxInventories|?{$_.LastWriteTime -gt $ExecStart.AddDays(-1)})) { $ViEsxsInfosCsv|Out-File -Path /mnt/wfs/inventory/ViEsxInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop } } catch { @@ -456,9 +460,11 @@ if ($ViServersList.count -gt 0) { if ($ViDatastoresInfos) { - if ($(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv")) { + $ViDsInventories = Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv" + + if ($ViDsInventories) { Write-Host "$((Get-Date).ToString("o")) [INFO] Rotating ViDsInventory.*.csv files ..." - $ExtraCsvFiles = Compare-Object $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv") $(Get-ChildItem "/mnt/wfs/inventory/ViDsInventory.*.csv"|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} + $ExtraCsvFiles = Compare-Object $ViDsInventories $($ViDsInventories|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} If ($ExtraCsvFiles) { try { Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false @@ -471,7 +477,7 @@ if ($ViServersList.count -gt 0) { try { Write-Host "$((Get-Date).ToString("o")) [INFO] Writing Datastore Inventory files ..." $ViDatastoresInfos|Export-Csv -NoTypeInformation -Path /mnt/wfs/inventory/ViDsInventory.csv -Force -ErrorAction Stop - if ($ExecStart.DayOfWeek -match "Monday" -and $ExecStart.Hour -eq "1") { + if ($ExecStart.DayOfWeek -match "Monday" -and !$($ViDsInventories|?{$_.LastWriteTime -gt $ExecStart.AddDays(-1)})) { $ViDatastoresInfos|Out-File -Path /mnt/wfs/inventory/ViDsInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -Force -ErrorAction Stop } } catch { From f4a66da4cadd072c3e127f1b725041cac6a22545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 5 Dec 2023 23:44:20 +0100 Subject: [PATCH 188/201] Create VbrConnect.ps1 --- opt/sexigraf/VbrConnect.ps1 | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 opt/sexigraf/VbrConnect.ps1 diff --git a/opt/sexigraf/VbrConnect.ps1 b/opt/sexigraf/VbrConnect.ps1 new file mode 100644 index 00000000..c5a7baa6 --- /dev/null +++ b/opt/sexigraf/VbrConnect.ps1 @@ -0,0 +1,32 @@ +#!/usr/bin/pwsh -NonInteractive -NoProfile -Command +# +param([Parameter (Mandatory=$true)] [string] $server, [Parameter (Mandatory=$true)] [string] $username, [Parameter (Mandatory=$true)] [string] $password) + +# https://communities.vmware.com/t5/VMware-PowerCLI-Discussions/PowerCLI-on-Debian-Stretch-The-type-initializer-for-VMware/m-p/451739#M10646 +Set-Content -Path Env:HOME -Value '/tmp' + +if (!$(Test-Connection -TargetName $server -TcpPort 9419 -TimeoutSeconds 2)) { + Write-Host "$server is not answering at TCP:9419 check if VeeamBackupRESTSvc is running" + exit 1 +} + +try { + $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} + $VbrBody = @{grant_type = "password";username = $username;password = $password;refresh_token = "";code = "";use_short_term_refresh = ""} + $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody + if ($VbrConnect.access_token) { + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $SessionRefresh = "vbr_" + $server.Replace(".","_") + ".dat" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefresh + Write-Host "Connected to $server" + } else { + Write-Host "Connection to $server failed!" + exit 1 + } +} catch { + # Invoke-RestMethod: Unable to read data from the transport connection: Connection reset by peer. + # https://helpcenter.veeam.com/docs/backup/vbr_rest/tls_certificate.html + Write-Host "$($Error[0])" + exit 1 +} \ No newline at end of file From 12268e5ce1bdd92bab208c59000ea4860d09029e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Tue, 5 Dec 2023 23:44:22 +0100 Subject: [PATCH 189/201] Create VbrPullStatistics.ps1 --- opt/sexigraf/VbrPullStatistics.ps1 | 477 +++++++++++++++++++++++++++++ 1 file changed, 477 insertions(+) create mode 100644 opt/sexigraf/VbrPullStatistics.ps1 diff --git a/opt/sexigraf/VbrPullStatistics.ps1 b/opt/sexigraf/VbrPullStatistics.ps1 new file mode 100644 index 00000000..1ee46cd3 --- /dev/null +++ b/opt/sexigraf/VbrPullStatistics.ps1 @@ -0,0 +1,477 @@ +#!/usr/bin/pwsh -Command +# +param([Parameter (Mandatory=$true)] [string] $Server, [Parameter (Mandatory=$true)] [string] $SessionFile, [Parameter (Mandatory=$false)] [string] $CredStore) + +$ScriptVersion = "0.9.44" + +$ExecStart = $(Get-Date).ToUniversalTime() +# $stopwatch = [system.diagnostics.stopwatch]::StartNew() + +$ErrorActionPreference = "SilentlyContinue" +$WarningPreference = "SilentlyContinue" +(Get-Process -Id $pid).PriorityClass = 'Idle' +$SexiVbrMutex = New-Object System.Threading.Mutex($false, "SexiVbrMutex") +# https://learn-powershell.net/2014/09/30/using-mutexes-to-write-data-to-the-same-logfile-across-processes-with-powershell/ +# https://github.com/ryan-leap/GreenMeansGoMutexDemo + +function SexiLogger { + param($Text2Log) + Write-Host "$((Get-Date).ToString("o")) $Text2Log" + $null = $SexiVbrMutex.WaitOne(500) + Add-Content -Path "/var/log/sexigraf/VbrPullStatistics.log" -Value "$((Get-Date).ToString("o")) $($Server) $Text2Log" + $SexiVbrMutex.ReleaseMutex() +} + +function AltAndCatchFire { + Param($ExitReason) + SexiLogger "[EROR] $ExitReason" + SexiLogger "[EROR] $($Error[0])" + SexiLogger "[EROR] Exit" + Stop-Transcript + exit +} + +function NameCleaner { + Param($NameToClean) + $NameToClean = $NameToClean -replace "[ .]","_" + [System.Text.NormalizationForm]$NormalizationForm = "FormD" + $NameToClean = $NameToClean.Normalize($NormalizationForm) + $NameToClean = $NameToClean -replace "[^[:ascii:]]","" -replace "[^A-Za-z0-9-_]","_" + return $NameToClean.ToLower() +} + + +try { + Start-Transcript -Path "/var/log/sexigraf/VbrPullStatistics.$($Server).log" -Append -Confirm:$false -Force -UseMinimalHeader + SexiLogger "[INFO] VbrPullStatistics v$ScriptVersion" +} catch { + SexiLogger "[EROR] VbrPullStatistics logging failure" + SexiLogger "[EROR] Exit" + exit +} + +try { + SexiLogger "[INFO] Importing Graphite PowerShell module ..." + Import-Module -Name /usr/local/share/powershell/Modules/Graphite-PowerShell-Functions/Graphite-Powershell.psm1 -Global -Force -SkipEditionCheck +} catch { + AltAndCatchFire "Powershell modules import failure" +} + +try { + SexiLogger "[INFO] Looking for another VbrPullStatistics for $Server ..." + $DupVbrPullStatisticsProcess = Get-PSHostProcessInfo|%{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '}|?{$_ -match "VbrPullStatistics" -and $_ -match "$Server"} + # https://github.com/PowerShell/PowerShell/issues/13944 + if (($DupVbrPullStatisticsProcess|Measure-Object).Count -gt 1) { + $DupVbrPullStatisticsProcessId = (Get-PSHostProcessInfo|?{$(Get-Content -LiteralPath "/proc/$($_.ProcessId)/cmdline") -replace "`0", ' '|?{$_ -match "$Server"}}).ProcessId[0] + $DupVbrPullStatisticsProcessTime = [INT32](ps -p $DupVbrPullStatisticsProcessId -o etimes).split()[-1] + if ($DupVbrPullStatisticsProcessTime -gt 300) { + SexiLogger "[WARN] VbrPullStatistics for $Server is already running for more than 5 minutes!" + SexiLogger "[WARN] Killing stunned VbrPullStatistics for $Server" + Stop-Process -Id $DupVbrPullStatisticsProcessId -Force + } else { + AltAndCatchFire "VbrPullStatistics for $Server is already running!" + } + } +} catch { + AltAndCatchFire "VbrPullStatistics process lookup failure" +} + +if ($SessionFile) { + try { + $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" + if ([DateTime]$(Get-Content -Path /tmp/$SessionSecretExpiration) -gt $ExecStart.AddMinutes(5)) { + $SessionToken = Get-Content -Path $SessionFile -ErrorAction Stop + SexiLogger "[INFO] SessionToken found in SessionFile, attempting connection to $Server ..." + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders + if ($($VbrJobsStates.data)) { + SexiLogger "[INFO] Connected to VBR REST API Server $Server" + } else { + SexiLogger "[WARN] Connection failure or no job state" + } + } else { + SexiLogger "[WARN] Token has expired or is about to ..." + } + } catch { + SexiLogger "[WARN] SessionToken or SessionSecretExpiration not found, invalid or connection failure" + } + + if (!$($VbrJobsStates.data)) { + SexiLogger "[WARN] Attempting token refresh ..." + try { + $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" + $SessionRefresh = Get-Content -Path /tmp/$SessionRefreshPath -ErrorAction Stop + $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} + $VbrBody = @{grant_type = "refresh_token";username = "";password = "";refresh_token = $SessionRefresh;code = "";use_short_term_refresh = ""} + $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody + if ($VbrConnect.access_token) { + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefreshPath + $VbrConnect.".expires".ToUniversalTime().tostring() | Out-File -FilePath /tmp/$SessionSecretExpiration + $SessionToken = $VbrConnect.access_token + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders + if (!$($VbrJobsStates.data)) { + SexiLogger "[WARN] Token refresh failed!" + SexiLogger "[WARN] Known issue on v11!" + # https://forums.veeam.com/restful-api-f30/how-to-handle-the-refresh-token-t74916.html + } + } else { + SexiLogger "[WARN] Token refresh failed!" + } + } catch { + SexiLogger "[WARN] Token refresh issue!" + } + } + + if (!$($VbrJobsStates.data)) { + SexiLogger "[WARN] Attempting explicit connection ..." + try { + $createstorexml = New-Object -TypeName XML + $createstorexml.Load($credstore) + $XPath = '//passwordEntry[server="' + $Server + '"]' + if ($(Select-XML -Xml $createstorexml -XPath $XPath)){ + $item = Select-XML -Xml $createstorexml -XPath $XPath + $CredStoreLogin = $item.Node.username + $CredStorePassword = [System.Text.Encoding]::Unicode.GetString([System.Convert]::FromBase64String($item.Node.password)) + } else { + AltAndCatchFire "No $Server entry in CredStore" + } + $VbrHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"} + $VbrBody = @{grant_type = "password";username = $CredStoreLogin;password = $CredStorePassword;refresh_token = "";code = "";use_short_term_refresh = ""} + $VbrConnect = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method POST -Uri $("https://" + $server + ":9419/api/oauth2/token") -Headers $VbrHeaders -ContentType "application/x-www-form-urlencoded" -Body $VbrBody + if ($VbrConnect.access_token) { + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $($VbrConnect.access_token)"} + $VbrJobsStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/jobs/states") -Headers $VbrAuthHeaders + if ($($VbrJobsStates.data)) { + SexiLogger "[INFO] Connected to VBR REST API Server $Server" + $SessionSecretName = "vbr_" + $server.Replace(".","_") + ".key" + $SessionRefreshPath = "vbr_" + $server.Replace(".","_") + ".dat" + $SessionSecretExpiration = "vbr_" + $server.Replace(".","_") + ".exp" + $VbrConnect.access_token | Out-File -FilePath /tmp/$SessionSecretName + $VbrConnect.refresh_token | Out-File -FilePath /tmp/$SessionRefreshPath + $VbrConnect.".expires".ToUniversalTime().tostring() | Out-File -FilePath /tmp/$SessionSecretExpiration + $SessionToken = $VbrConnect.access_token + } else { + AltAndCatchFire "Jobs States check failed, no job state or check the user permissions!" + } + } else { + AltAndCatchFire "Explicit connection failed, check the stored credentials!" + } + } catch { + AltAndCatchFire "Explicit connection failed, check the stored credentials!" + } + } +} else { + AltAndCatchFire "No SessionFile somehow ..." +} + +if ($($VbrJobsStates.data)) { + $VbrAuthHeaders = @{"accept" = "application/json";"x-api-version" = "1.0-rev1"; "Authorization" = "Bearer $SessionToken"} + $VbrDataTable = @{} + $vbrserver_name = NameCleaner $Server + SexiLogger "[INFO] Start processing VBR Server $Server ..." + + $VbrJobsStatesTable = @{} + foreach ($VbrJobState in $VbrJobsStates.data) { + try { + $VbrJobsStatesTable.add($VbrJobState.id,$VbrJobState) + } catch {} + $job_name = NameCleaner $VbrJobState.name + if ($VbrJobState.status -eq "running") { + $VbrJobStateStatus = 0 + } elseif ($VbrJobState.status -eq "inactive") { + $VbrJobStateStatus = 1 + } elseif ($VbrJobState.status -eq "disabled") { + $VbrJobStateStatus = 2 + } else { + $VbrJobStateStatus = 3 + } + $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.status"] = $VbrJobStateStatus + } + + try { + SexiLogger "[INFO] VBR repositories states collect ..." + $VbrRepositoriesStates = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupInfrastructure/repositories/states") -Headers $VbrAuthHeaders + } catch { + SexiLogger "[EROR] repositories states collect failure" + SexiLogger "[EROR] $($Error[0])" + } + + if ($VbrRepositoriesStates.data) { + SexiLogger "[INFO] VBR repositories processing ..." + $VbrRepositoryTable = @{} + foreach ($VbrRepository in $VbrRepositoriesStates.data) { + $VbrRepositoryName = NameCleaner $VbrRepository.name + + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.capacityGB"] = $VbrRepository.capacityGB + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.freeGB"] = $VbrRepository.freeGB + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.usedSpaceGB"] = $VbrRepository.usedSpaceGB + $VbrDataTable["veeam.vbr.$vbrserver_name.repo.$VbrRepositoryName.RealUsedPct"] = $($VbrRepository.usedSpaceGB * 100 / $($VbrRepository.usedSpaceGB + $VbrRepository.freeGB )) + + try { + $VbrRepositoryTable.add($VbrRepository.id,$VbrRepository) + } catch {} + } + } else { + SexiLogger "[WARN] No repositories ?!" + } + + try { + SexiLogger "[INFO] VBR ScaleOutRepositories collect ..." + $VbrScaleOutRepositories = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupInfrastructure/scaleOutRepositories") -Headers $VbrAuthHeaders + } catch { + SexiLogger "[EROR] ScaleOutRepositories collect failure" + SexiLogger "[EROR] $($Error[0])" + } + + if ($VbrScaleOutRepositories.data) { + SexiLogger "[INFO] SOBR performanceTier processing ..." + foreach ($VbrScaleOutRepository in $VbrScaleOutRepositories.data) { + try { + $VbrScaleOutRepositoryName = NameCleaner $VbrScaleOutRepository.name + + $VbrSobrNotNormal = $VbrScaleOutRepository.performanceTier.performanceExtents|?{$_.status -ne "Normal"} + + if ($VbrSobrNotNormal.id) { + $VbrSobrNotNormalRepos = $VbrRepositoryTable[$VbrSobrNotNormal.id] + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.capacityGB"] += $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property capacityGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] += $($VbrSobrNotNormalRepos|Measure-Object -Sum -Property usedSpaceGB).Sum + } + + $VbrSobrNormal = $VbrScaleOutRepository.performanceTier.performanceExtents|?{$_.status -eq "Normal"} + if ($VbrSobrNormal.id) { + $VbrSobrNormalRepos = $VbrRepositoryTable[$VbrSobrNormal.id] + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.capacityGB"] += $($VbrSobrNormalRepos|Measure-Object -Sum -Property capacityGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] += $($VbrSobrNormalRepos|Measure-Object -Sum -Property usedSpaceGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"] = $($VbrSobrNormalRepos|Measure-Object -Sum -Property freeGB).Sum + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.RealUsedPct"] = $($VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] * 100 / $($VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.usedSpaceGB"] + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"])) + } else { + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.freeGB"] = 0 + $VbrDataTable["veeam.vbr.$vbrserver_name.sobr.$VbrScaleOutRepositoryName.RealUsedPct"] = 100 + } + } catch { + SexiLogger "[EROR] Issue processing SOBR $($VbrScaleOutRepository.name)" + SexiLogger "[EROR] $($Error[0])" + } + } + } else { + SexiLogger "[INFO] No SOBR" + } + + try { + SexiLogger "[INFO] VBR 5min old objectRestorePoints collect ..." + # $VbrSessions5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + $VbrObjectRestorePoints5 = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/objectRestorePoints?platformNameFilter=VmWare&limit=999&createdAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + } catch { + SexiLogger "[EROR] objectRestorePoints collect failure" + SexiLogger "[EROR] $($Error[0])" + } + + # $VbrVmwareServers = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/inventory/vmware/hosts") -Headers $VbrAuthHeaders + + + if ($VbrObjectRestorePoints5.data) { + # SexiLogger "[INFO] VBR backupObjects collect ..." + # $VbrBackupObjects5 = @{} + # foreach ($VbrObjectRestorePoint5 in $VbrObjectRestorePoints5.data) { + # try { + # $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&nameFilter=" + $($VbrObjectRestorePoint5.name)) -Headers $VbrAuthHeaders + # if ($VbrBackupObjects.data[0].type -eq "VM" -and !$VbrBackupObjects5[$VbrBackupObjects.data[0].name]) { + # $VbrBackupObjects5.Add($VbrBackupObjects.data[0].name,$VbrBackupObjects[0].data) + # } + # } catch { + # SexiLogger "[EROR] backupObjects collect failure" + # SexiLogger "[EROR] $($Error[0])" + # } + # } # Too Slow !!! + + try { + SexiLogger "[INFO] VBR backupObjects collect ..." + $VbrBackupObjects = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backupObjects?platformNameFilter=VmWare&limit=9999") -Headers $VbrAuthHeaders # TODO limit=9999+ ? + if ($VbrBackupObjects.data) { + SexiLogger "[INFO] Building VBR backupObjects table ..." + $VbrBackupObjectsTable = @{} + foreach ($VbrBackupObject in $VbrBackupObjects.data) { + try { + $VbrBackupObjectsTable.add($VbrBackupObject.name,$VbrBackupObject) + } catch {} + } + } + } catch { + SexiLogger "[EROR] backupObjects collect failure" + SexiLogger "[EROR] $($Error[0])" + } + + $VbrObjectRestorePoints5SessionsId = $VbrObjectRestorePoints5.data.backupId|Select-Object -Unique + + SexiLogger "[INFO] VBR Sessions collect ..." + $VbrSessions5 = @{} + foreach ($VbrObjectRestorePoints5SessionId in $VbrObjectRestorePoints5SessionsId) { + try { + $VbrObjectRestorePoints5Session = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/backups/" + $VbrObjectRestorePoints5SessionId) -Headers $VbrAuthHeaders + if ($VbrObjectRestorePoints5Session|?{$_.jobId -ne "00000000-0000-0000-0000-000000000000"}) { + $VbrSessions5.Add($VbrObjectRestorePoints5SessionId,$VbrObjectRestorePoints5Session) + } + } catch { + SexiLogger "[EROR] backupObjects collect failure" + SexiLogger "[EROR] $($Error[0])" + } + } + + if (Test-Path /mnt/wfs/inventory/ViVmInventory.csv) { + SexiLogger "[INFO] Importing VM inventory ..." + try { + $ViVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/ViVmInventory.csv -ErrorAction Stop) + } catch { + SexiLogger "[EROR] ViVmInventory import issue" + SexiLogger "[EROR] $($Error[0])" + } + + if ($ViVmInventory) { + $ViVmInventoryTable = @{} + foreach ($ViVm in $ViVmInventory) { + try { + $ViVmInventoryTable.Add($ViVm.vm,$ViVm) + } catch {} + } + + if (Test-Path /mnt/wfs/inventory/VbrInv_$($Server).csv) { + SexiLogger "[INFO] Importing $($Server) VBR inventory ..." + try { + $VbrVmInventory = $(Import-Csv -Path /mnt/wfs/inventory/VbrInv_$($Server).csv -ErrorAction Stop) + } catch { + SexiLogger "[EROR] VbrVmInventory import issue" + SexiLogger "[EROR] $($Error[0])" + } + if ($ViVmInventory) { + $VbrVmInventoryTable = @{} + foreach ($VbrVm in $VbrVmInventory) { + try { + if ($ViVmInventoryTable[$VbrVm.vm]) { + $VbrVmInventoryTable.Add($VbrVm.vm,$VbrVm) + } + } catch {} + } + } + } else { + $VbrVmInventoryTable = @{} + } + + SexiLogger "[INFO] Building $($Server) VBR DataTable and inventory ..." + foreach ($VbrObjectRestorePoint in $VbrObjectRestorePoints5.data) { + if ($VbrJobsStatesTable[$VbrSessions5[$VbrObjectRestorePoint.backupId].jobId]) { + $job_name = NameCleaner $VbrSessions5[$VbrObjectRestorePoint.backupId].name + $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.objectRestorePoints"] ++ + $vm_name = NameCleaner $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name + if ($ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name]) { + $vcenter_name = NameCleaner $ViVmInventoryTable[$vm_name].vCenter + if ($ViVmInventoryTable[$vm_name].Cluster) { + $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].Cluster + } else { + $cluster_name = NameCleaner $ViVmInventoryTable[$vm_name].ESX + } + $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.objectRestorePoints"] ++ + $VbrDataTable["veeam.vi.$vcenter_name.$cluster_name.vm.$vm_name.restorePointsCount"] = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount + + $VbrObjectInventoryInfo = "" | Select-Object VbrServer, JobName, vCenter, Cluster, VM, RestorePointsCount, LastRestorePoint + $VbrObjectInventoryInfo.VbrServer = $Server + $VbrObjectInventoryInfo.JobName = $VbrSessions5[$VbrObjectRestorePoint.backupId].name + $VbrObjectInventoryInfo.vCenter = $ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name].vCenter + $VbrObjectInventoryInfo.Cluster = $ViVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name].Cluster + $VbrObjectInventoryInfo.VM = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name + $VbrObjectInventoryInfo.RestorePointsCount = $VbrBackupObjectsTable[$VbrObjectRestorePoint.name].restorePointsCount + $VbrObjectInventoryInfo.LastRestorePoint = $VbrObjectRestorePoint.creationTime + if ($VbrVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name]) { + $VbrVmInventoryTable[$VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name] = $VbrObjectInventoryInfo + } else { + $VbrVmInventoryTable.add($VbrBackupObjectsTable[$VbrObjectRestorePoint.name].name,$VbrObjectInventoryInfo) + } + } + } + } + + if ($VbrVmInventoryTable.Values) { + SexiLogger "[INFO] Exporting $($Server) VBR inventory ..." + try { + $VbrVmInventoryTable.Values|Sort-Object VM|Export-Csv /mnt/wfs/inventory/VbrInv_$($Server).csv -ErrorAction Stop -Force + } catch { + SexiLogger "[EROR] VbrVmInventory import issue" + SexiLogger "[EROR] $($Error[0])" + } + + $VbrVmInventories = Get-ChildItem "/mnt/wfs/inventory/VbrVmInventory.*.csv" + + if ($VbrVmInventories) { + SexiLogger "[INFO] Rotating VbrVmInventory.*.csv files ..." + $ExtraCsvFiles = Compare-Object $VbrVmInventories $($VbrVmInventories|Sort-Object LastWriteTime | Select-Object -Last 10) -property FullName | ?{$_.SideIndicator -eq "<="} + If ($ExtraCsvFiles) { + try { + Get-ChildItem $ExtraCsvFiles.FullName | Remove-Item -Force -Confirm:$false + } catch { + AltAndCatchFire "Cannot remove extra csv files" + } + } + } + + SexiLogger "[INFO] Import/Export VBR inventories ..." + try { + $VbrVmInventoryCsv = Import-Csv $(Get-ChildItem /mnt/wfs/inventory/VbrInv_*.csv).fullname | Sort-Object LastRestorePoint -Descending + $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.csv -ErrorAction Stop -Force + if ($ExecStart.DayOfWeek -match "Monday" -and !$($VbrVmInventories|?{$_.LastWriteTime -gt $ExecStart.AddDays(-1)})) { + $VbrVmInventoryCsv | Export-Csv /mnt/wfs/inventory/VbrVmInventory.$((Get-Date).ToString("yyyy.MM.dd_hh.mm.ss")).csv -ErrorAction Stop -Force + } + } catch { + SexiLogger "[EROR] Import/Export VBR inventories issue" + SexiLogger "[EROR] $($Error[0])" + } + } else { + SexiLogger "[EROR] No VbrVmInventoryTable values ?!" + } + } else { + SexiLogger "[EROR] ViVmInventory import issue" + } + } else { + SexiLogger "[EROR] No ViVmInventory" + } + } + + try { + SexiLogger "[INFO] VBR ended sessions collect ..." + $VbrEndedSessions = Invoke-RestMethod -SkipHttpErrorCheck -SkipCertificateCheck -Method GET -Uri $("https://" + $server + ":9419/api/v1/sessions?endedAfterFilter=" + $(($ExecStart.AddMinutes(-5)).ToString("o"))) -Headers $VbrAuthHeaders + } catch { + SexiLogger "[EROR] Sessions collect failure" + SexiLogger "[EROR] $($Error[0])" + } + + if ($VbrEndedSessions.data) { + SexiLogger "[INFO] Processing VBR ended sessions ..." + foreach ($VbrEndedSession in $VbrEndedSessions.data) { + $job_name = NameCleaner $VbrEndedSession.name + if ($VbrEndedSession.result.result -eq "Success") { + $VbrEndedSessionResult = 0 + } elseif ($VbrEndedSession.result.result -eq "Warning") { + $VbrEndedSessionResult = 1 + } elseif ($VbrEndedSession.result.result -eq "Failed") { + $VbrEndedSessionResult = 2 + } else { + $VbrEndedSessionResult = 3 + } + $VbrDataTable["veeam.vbr.$vbrserver_name.job.$job_name.result"] = $VbrEndedSessionResult + } + } + + $VbrDataTable["veeam.vbr.$vbrserver_name.exec.duration"] = $($(Get-Date).ToUniversalTime() - $ExecStart).TotalSeconds + SexiLogger "[INFO] Sending veeam data to Graphite for VBR server $Server ..." + Send-BulkGraphiteMetrics -CarbonServer 127.0.0.1 -CarbonServerPort 2003 -Metrics $VbrDataTable -DateTime $ExecStart + + SexiLogger "[INFO] End of VBR server $server processing ..." + +} else { + SexiLogger "[INFO] No Running Jobs on VBR server $server ..." + SexiLogger "[INFO] Exit" + Stop-Transcript + exit +} \ No newline at end of file From 19f825d110bb236c9f5a09eae6d710323a21e2bb Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 21 Dec 2023 15:40:38 +0100 Subject: [PATCH 190/201] Update VMware_Evo_Version.json --- .../grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json b/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json index 838af96c..30519dad 100644 --- a/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json +++ b/var/lib/grafana/dashboards/VMware_vCenter/VMware_Evo_Version.json @@ -347,14 +347,14 @@ "list": [ { "allFormat": "wildcard", - "allValue": null, + "allValue": "*", "current": {}, "datasource": null, "definition": "vi.*", "description": null, "error": null, "hide": 0, - "includeAll": false, + "includeAll": true, "label": null, "multi": true, "multiFormat": "glob", From 453528dac4d9068998d1d774f56aa8e6a3066c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 28 Dec 2023 00:57:55 +0100 Subject: [PATCH 191/201] Update VMware_All_Standalone_ESX_VM_Stats.json --- .../VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json index 2950da2c..35c2e460 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_VM_Stats.json @@ -652,7 +652,7 @@ "targets": [ { "refId": "A", - "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskIOPS, '$N')), 5)", + "target": "aliasByNode(sortByMaxima(highest$view(esx.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskIOPS, '$N')), 5)", "textEditor": true } ], From 31b7b741db64e32a4f68c1527e0678eaa86f0da2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 28 Dec 2023 00:57:57 +0100 Subject: [PATCH 192/201] Update VMware_All_Standalone_ESX_Top_N_VM_Stats.json --- .../VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json index 971135b2..44390e10 100644 --- a/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json +++ b/var/lib/grafana/dashboards/VMware_VM/VMware_All_Standalone_ESX_Top_N_VM_Stats.json @@ -652,7 +652,7 @@ "targets": [ { "refId": "A", - "target": "aliasByNode(sortByMaxima(highest$view(vmw.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskIOPS, '$N')), 5)", + "target": "aliasByNode(sortByMaxima(highest$view(esx.$vcenter.$datacentre.$esx.vm.$vm.fatstats.diskIOPS, '$N')), 5)", "textEditor": true } ], From 8ee3cf3f5a4e627375d6d3fe357b83539b0445de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 28 Dec 2023 00:58:00 +0100 Subject: [PATCH 193/201] Update ViPullStatistics.ps1 --- opt/sexigraf/ViPullStatistics.ps1 | 52 +++++++++++++++---------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 7da93229..8dc68e8f 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([parameter (Mandatory=$true)] [string] $Server, [parameter (Mandatory=$true)] [string] $SessionFile, [parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1033" +$ScriptVersion = "0.9.1034" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -572,7 +572,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } - if ($vcenter_clusters_h.Keys) { + if ($vcenter_clusters_h.keys.count -gt 0) { $ClusterMultiMetrics = @( "vmop.numSVMotion.latest", "vmop.numXVMotion.latest" @@ -582,7 +582,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { SexiLogger "[INFO] All Clusters multi metrics collected in $($ClusterMultiStatsTime.TotalSeconds) sec for vCenter $vcenter_name" } catch { SexiLogger "[EROR] $($Error[0])" - SexiLogger "[EROR] VM MultiQueryPerf failure" + SexiLogger "[EROR] VM MultiQueryPerf300 failure" } } @@ -1956,7 +1956,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { # if ($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value] -and $VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_standalone_host_vm.moref.value]) { # $vcenter_standalone_host_vm_net_iops = $($($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum + $($VmMultiStats[$PerfCounterTable["net.packetsRx.summation"]][$vcenter_standalone_host_vm.moref.value][$($VmMultiStats[$PerfCounterTable["net.packetsTx.summation"]][$vcenter_standalone_host_vm.moref.value]).Keys]|Measure-Object -Sum).Sum) / 300 - # $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.netIOPS", $vcenter_standalone_host_vm_net_iops) + # $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.vm.$vcenter_standalone_host_vm_name.fatstats.netIOPS", $vcenter_standalone_host_vm_net_iops) # } if ($VmMultiStats[$PerfCounterTable["net.usage.average"]][$vcenter_standalone_host_vm.moref.value][""]) { @@ -2318,13 +2318,13 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { SexiLogger "[EROR] $($Error[0])" } - $UnamagedResourcePoolPrivateMemory = 0 - $UnamagedResourcePoolSharedMemory = 0 - $UnamagedResourcePoolBalloonedMemory = 0 - $UnamagedResourcePoolCompressedMemory = 0 - $UnamagedResourcePoolSwappedMemory = 0 - $UnamagedResourcePoolGuestMemoryUsage = 0 - $UnamagedResourcePoolConsumedOverheadMemory = 0 + $UnmanagedResourcePoolPrivateMemory = 0 + $UnmanagedResourcePoolSharedMemory = 0 + $UnmanagedResourcePoolBalloonedMemory = 0 + $UnmanagedResourcePoolCompressedMemory = 0 + $UnmanagedResourcePoolSwappedMemory = 0 + $UnmanagedResourcePoolGuestMemoryUsage = 0 + $UnmanagedResourcePoolConsumedOverheadMemory = 0 $unmanaged_host_vms_vcpus = 0 $unmanaged_host_vms_vram = 0 @@ -2458,17 +2458,17 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($unmanaged_host_vm.summary.quickStats.balloonedMemory -gt 0) { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.quickstats.BalloonedMemory", $unmanaged_host_vm.summary.quickStats.balloonedMemory) - $UnamagedResourcePoolBalloonedMemory += $unmanaged_host_vm.summary.quickStats.balloonedMemory + $UnmanagedResourcePoolBalloonedMemory += $unmanaged_host_vm.summary.quickStats.balloonedMemory } if ($unmanaged_host_vm.summary.quickStats.compressedMemory -gt 0) { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.quickstats.CompressedMemory", $unmanaged_host_vm.summary.quickStats.compressedMemory) - $UnamagedResourcePoolCompressedMemory += $unmanaged_host_vm.summary.quickStats.compressedMemory + $UnmanagedResourcePoolCompressedMemory += $unmanaged_host_vm.summary.quickStats.compressedMemory } if ($unmanaged_host_vm.summary.quickStats.swappedMemory -gt 0) { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.quickstats.SwappedMemory", $unmanaged_host_vm.summary.quickStats.swappedMemory) - $UnamagedResourcePoolSwappedMemory += $unmanaged_host_vm.summary.quickStats.swappedMemory + $UnmanagedResourcePoolSwappedMemory += $unmanaged_host_vm.summary.quickStats.swappedMemory } if ($VmMultiStats[$PerfCounterTable["cpu.ready.summation"]][$unmanaged_host_vm.moref.value][""]) { @@ -2493,7 +2493,7 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($VmMultiStats[$PerfCounterTable["disk.maxTotalLatency.latest"]][$unmanaged_host_vm.moref.value][""]) { $unmanaged_host_vm_disk_latency = $VmMultiStats[$PerfCounterTable["disk.maxTotalLatency.latest"]][$unmanaged_host_vm.moref.value][""] $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.maxTotalLatency", $unmanaged_host_vm_disk_latency) - } + }Unmanaged if ($VmMultiStats[$PerfCounterTable["virtualdisk.write.average"]][$unmanaged_host_vm.moref.value][""] -ge 0 -and $VmMultiStats[$PerfCounterTable["virtualdisk.read.average"]][$unmanaged_host_vm.moref.value][""] -ge 0) { $unmanaged_host_vm_disk_usage = $VmMultiStats[$PerfCounterTable["virtualdisk.write.average"]][$unmanaged_host_vm.moref.value][""] + $VmMultiStats[$PerfCounterTable["virtualdisk.read.average"]][$unmanaged_host_vm.moref.value][""] @@ -2519,10 +2519,10 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.vm.$unmanaged_host_vm_name.fatstats.netUsage", 0) } - if ($unmanaged_host_vm.summary.quickStats.privateMemory -gt 0) {$UnamagedResourcePoolPrivateMemory += $unmanaged_host_vm.summary.quickStats.privateMemory} - if ($unmanaged_host_vm.summary.quickStats.GuestMemoryUsage -gt 0) {$UnamagedResourcePoolGuestMemoryUsage += $unmanaged_host_vm.summary.quickStats.GuestMemoryUsage} - if ($unmanaged_host_vm.summary.quickStats.SharedMemory -gt 0) {$UnamagedResourcePoolSharedMemory += $unmanaged_host_vm.summary.quickStats.SharedMemory} - if ($unmanaged_host_vm.summary.quickStats.ConsumedOverheadMemory -gt 0) {$UnamagedResourcePoolConsumedOverheadMemory += $unmanaged_host_vm.summary.quickStats.ConsumedOverheadMemory} + if ($unmanaged_host_vm.summary.quickStats.privateMemory -gt 0) {$UnmanagedResourcePoolPrivateMemory += $unmanaged_host_vm.summary.quickStats.privateMemory} + if ($unmanaged_host_vm.summary.quickStats.GuestMemoryUsage -gt 0) {$UnmanagedResourcePoolGuestMemoryUsage += $unmanaged_host_vm.summary.quickStats.GuestMemoryUsage} + if ($unmanaged_host_vm.summary.quickStats.SharedMemory -gt 0) {$UnmanagedResourcePoolSharedMemory += $unmanaged_host_vm.summary.quickStats.SharedMemory} + if ($unmanaged_host_vm.summary.quickStats.ConsumedOverheadMemory -gt 0) {$UnmanagedResourcePoolConsumedOverheadMemory += $unmanaged_host_vm.summary.quickStats.ConsumedOverheadMemory} } catch { SexiLogger "[EROR] VM $unmanaged_host_vm_name metric issue on unmanaged host $unmanaged_host_name" @@ -2536,13 +2536,13 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { } try { - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.ballooned", $UnamagedResourcePoolBalloonedMemory) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.compressed", $UnamagedResourcePoolCompressedMemory) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.consumedOverhead", $UnamagedResourcePoolConsumedOverheadMemory) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.guest", $UnamagedResourcePoolGuestMemoryUsage) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.private", $UnamagedResourcePoolPrivateMemory) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.shared", $UnamagedResourcePoolSharedMemory) - $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.swapped", $UnamagedResourcePoolSwappedMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.ballooned", $UnmanagedResourcePoolBalloonedMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.compressed", $UnmanagedResourcePoolCompressedMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.consumedOverhead", $UnmanagedResourcePoolConsumedOverheadMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.guest", $UnmanagedResourcePoolGuestMemoryUsage) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.private", $UnmanagedResourcePoolPrivateMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.shared", $UnmanagedResourcePoolSharedMemory) + $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.quickstats.mem.swapped", $UnmanagedResourcePoolSwappedMemory) $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.runtime.vm.total", $($unmanaged_host_vms_on + $unmanaged_host_vms_off)) $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.runtime.vm.on", $unmanaged_host_vms_on) $unmanaged_host_h.add("esx.$vcenter_name.$unmanaged_host_dc_name.$unmanaged_host_name.runtime.vm.dead", $unmanaged_host_vms_dead) From 96e7b8b0f8ebf496d7dd03e7fceb5fdcbf4716bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Thu, 28 Dec 2023 01:20:19 +0100 Subject: [PATCH 194/201] Update ViPullStatistics.ps1 --- opt/sexigraf/ViPullStatistics.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/opt/sexigraf/ViPullStatistics.ps1 b/opt/sexigraf/ViPullStatistics.ps1 index 8dc68e8f..2795e715 100644 --- a/opt/sexigraf/ViPullStatistics.ps1 +++ b/opt/sexigraf/ViPullStatistics.ps1 @@ -2,7 +2,7 @@ # param([parameter (Mandatory=$true)] [string] $Server, [parameter (Mandatory=$true)] [string] $SessionFile, [parameter (Mandatory=$false)] [string] $CredStore) -$ScriptVersion = "0.9.1034" +$ScriptVersion = "0.9.1035" $ExecStart = $(Get-Date).ToUniversalTime() # $stopwatch = [system.diagnostics.stopwatch]::StartNew() @@ -1713,7 +1713,8 @@ if ($ServiceInstance.Content.About.ApiType -match "VirtualCenter") { if ($vcenter_standalone_host_datastore.summary.type -notmatch "vsan") { $vcenter_standalone_host_datastore_uuid = $vcenter_standalone_host_datastore.summary.url.split("/")[-2] - $vcenter_standalone_host_datastore_latency = $HostMultiStats[$PerfCounterTable["datastore.sizeNormalizedDatastoreLatency.latest"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_datastore_uuid] + $vcenter_standalone_host_datastore_latency_raw = $HostMultiStats[$PerfCounterTable["datastore.totalWriteLatency.average"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_datastore_uuid] + $vcenter_standalone_host_datastore_latency = $(GetMedian $vcenter_standalone_host_datastore_latency_raw) * 1000 $vcenter_standalone_host_h.add("esx.$vcenter_name.$vcenter_standalone_host_dc_name.$vcenter_standalone_host_name.datastore.$vcenter_standalone_host_datastore_name.iorm.sizeNormalizedDatastoreLatency", $vcenter_standalone_host_datastore_latency) $vcenter_standalone_host_datastore_iops_w = $HostMultiStats[$PerfCounterTable["datastore.numberWriteAveraged.average"]][$vcenter_standalone_host.moref.value][$vcenter_standalone_host_datastore_uuid] From 5752964c80dc2aafa4fca1d6296cfa9126bf03f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 17 Jan 2024 15:31:57 +0100 Subject: [PATCH 195/201] Update PullGuestInfo.ps1 --- opt/sexigraf/PullGuestInfo.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opt/sexigraf/PullGuestInfo.ps1 b/opt/sexigraf/PullGuestInfo.ps1 index 65de4fa1..cf2bf2af 100644 --- a/opt/sexigraf/PullGuestInfo.ps1 +++ b/opt/sexigraf/PullGuestInfo.ps1 @@ -63,6 +63,10 @@ try { if ($VmwCmdOvfEnvGuest["guestinfo.dns"] -and ([regex]::match($($VmwCmdOvfEnvGuest['guestinfo.dns']), '^([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$')).Success) { $EtcNetworkInterfaces += " dns-nameservers $($VmwCmdOvfEnvGuest['guestinfo.dns'])" + } elseif ($VmwCmdOvfEnvGuest["guestinfo.dns"] -and ([regex]::match($($VmwCmdOvfEnvGuest['guestinfo.dns']), '^([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3}) ([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$')).Success) { + $EtcNetworkInterfaces += " dns-nameservers $($VmwCmdOvfEnvGuest['guestinfo.dns'])" + } elseif ($VmwCmdOvfEnvGuest["guestinfo.dns"] -and ([regex]::match($($VmwCmdOvfEnvGuest['guestinfo.dns']), '^([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3}) ([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3}) ([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})\.([0-9]{1,3})$')).Success) { + $EtcNetworkInterfaces += " dns-nameservers $($VmwCmdOvfEnvGuest['guestinfo.dns'])" } if ($VmwCmdOvfEnvGuest["guestinfo.domain"] -and ([regex]::match($($VmwCmdOvfEnvGuest['guestinfo.domain']), '([a-zA-Z]+)')).Success) { From df5ab51b7a5892960786fa7e5027a1b38a475b52 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 18 Jan 2024 10:54:20 +0100 Subject: [PATCH 196/201] Update graphite.conf #275 --- etc/apache2/sites-available/graphite.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/apache2/sites-available/graphite.conf b/etc/apache2/sites-available/graphite.conf index dbd7f29f..ea1d4a14 100644 --- a/etc/apache2/sites-available/graphite.conf +++ b/etc/apache2/sites-available/graphite.conf @@ -133,7 +133,7 @@ ExtendedStatus On RewriteCond %{HTTP_REFERER} /sexiadmin/ [NC] RewriteRule .* - [L] RewriteRule .* - [F] - Options Indexes + Options Indexes FollowSymLinks AllowOverride All Require all granted Allow from all From d57bd5873009e593114e4ce97c9b98ba9c63971a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Wed, 24 Jan 2024 23:29:39 +0100 Subject: [PATCH 197/201] Update home.json --- usr/share/grafana/public/dashboards/home.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/share/grafana/public/dashboards/home.json b/usr/share/grafana/public/dashboards/home.json index 37d2638e..96ff0249 100644 --- a/usr/share/grafana/public/dashboards/home.json +++ b/usr/share/grafana/public/dashboards/home.json @@ -1,5 +1,5 @@ { - "editable": true, + "editable": false, "gnetId": null, "graphTooltip": 1, "id": 62, From 7b249efff0520290c66892172d8a9a554d2f750d Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Thu, 25 Jan 2024 14:33:30 +0100 Subject: [PATCH 198/201] Update header.php --- var/www/admin/header.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/admin/header.php b/var/www/admin/header.php index 52bfd72b..cd27aa23 100644 --- a/var/www/admin/header.php +++ b/var/www/admin/header.php @@ -39,7 +39,7 @@
  • Veeam Credential Store
  • Package Updater
  • House Cleaner
    • -
  • Refresh Inventory
    • +
  • Inventory Refresh & History
  • Log Viewer
  • Export / Import
    • From 9fa38c14cc16f8fdb6cb35f9849d72be6a34b20e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20SCHITZ?= Date: Sun, 28 Jan 2024 21:03:16 +0100 Subject: [PATCH 199/201] Update refresh-inventory.php --- var/www/admin/refresh-inventory.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/admin/refresh-inventory.php b/var/www/admin/refresh-inventory.php index c741494f..d91c15fc 100644 --- a/var/www/admin/refresh-inventory.php +++ b/var/www/admin/refresh-inventory.php @@ -8,7 +8,7 @@

    Refresh VI Offline Inventory Notes

      -
    • The Static Offline Inventory is automatically schedule to be updated every 1 hour.
      • +
    • The Static VI Offline Inventory is automatically updated every hour.
    • If you want to force a refresh, you can use this section to perform update
    From 8ab2edd1e07a9047d68b9cfb3d56b4a61e1a2fad Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Mon, 5 Feb 2024 15:32:34 +0100 Subject: [PATCH 200/201] Update SexiGraf_Pull_Exec_Time.json --- .../grafana/dashboards/SexiGraf/SexiGraf_Pull_Exec_Time.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Pull_Exec_Time.json b/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Pull_Exec_Time.json index cc11bb5e..f98626e3 100644 --- a/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Pull_Exec_Time.json +++ b/var/lib/grafana/dashboards/SexiGraf/SexiGraf_Pull_Exec_Time.json @@ -78,7 +78,7 @@ { "refCount": 0, "refId": "D", - "target": "aliasByNode(veeam.vbr.$VbrServer.exec.duration, 2)", + "target": "aliasByNode(veeam.vbr.$VbrServer.exec.duration, 2, 1)", "textEditor": false } ], From ebde157cb535493bfce2798e815942f64bd8c2d6 Mon Sep 17 00:00:00 2001 From: Hypervisor Date: Fri, 9 Feb 2024 10:29:43 +0100 Subject: [PATCH 201/201] Update refresh-inventory.php --- var/www/admin/refresh-inventory.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/www/admin/refresh-inventory.php b/var/www/admin/refresh-inventory.php index d91c15fc..5c82520d 100644 --- a/var/www/admin/refresh-inventory.php +++ b/var/www/admin/refresh-inventory.php @@ -32,7 +32,7 @@

    Warning: Confirmation needed!

    - You are about to force inventory update. It should be use only for DEBUG purpose as it is already scheduled to run hourly.
    The process itself can take a few seconds (or minutes depending on your platform size). Are you sure about this? We mean, really sure?
    + You are about to force VI inventory update. It should be use only for DEBUG purpose as it is already scheduled to run hourly.
    The process itself can take a few seconds (or minutes depending on your platform size). Are you sure about this? We mean, really sure?

    '; @@ -50,7 +50,7 @@ } } else { echo '
    -

    +

    '; }
    vCenter/ESX address 
    " . $lineObjects[1] . ' ***********  * *