diff --git a/src/Player.Vm.Api/Domain/Vsphere/Models/VsphereAggregate.cs b/src/Player.Vm.Api/Domain/Vsphere/Models/VsphereAggregate.cs new file mode 100644 index 0000000..7101b03 --- /dev/null +++ b/src/Player.Vm.Api/Domain/Vsphere/Models/VsphereAggregate.cs @@ -0,0 +1,18 @@ +// Copyright 2022 Carnegie Mellon University. All Rights Reserved. +// Released under a MIT (SEI)-style license. See LICENSE.md in the project root for license information. + +using VimClient; + +namespace Player.Vm.Api.Domain.Vsphere.Models; + +public class VsphereAggregate +{ + public VsphereAggregate(VsphereConnection connection, ManagedObjectReference machineReference) + { + this.Connection = connection; + this.MachineReference = machineReference; + } + + public VsphereConnection Connection { get; set; } + public ManagedObjectReference MachineReference { get; set; } +} diff --git a/src/Player.Vm.Api/Domain/Vsphere/Models/VsphereConnection.cs b/src/Player.Vm.Api/Domain/Vsphere/Models/VsphereConnection.cs new file mode 100644 index 0000000..1dbed48 --- /dev/null +++ b/src/Player.Vm.Api/Domain/Vsphere/Models/VsphereConnection.cs @@ -0,0 +1,549 @@ +// Copyright 2024 Carnegie Mellon University. All Rights Reserved. +// Released under a MIT (SEI)-style license. See LICENSE.md in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.ServiceModel; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Player.Vm.Api.Domain.Vsphere.Extensions; +using Player.Vm.Api.Domain.Vsphere.Options; +using VimClient; + +namespace Player.Vm.Api.Domain.Vsphere.Models; + +public class VsphereConnection +{ + public VimPortTypeClient Client; + public ServiceContent Sic; + public UserSession Session; + public ManagedObjectReference Props; + public string Address + { + get + { + return Host.Address; + } + } + public bool Enabled + { + get + { + return Host.Enabled; + } + } + + public VsphereHost Host; + public VsphereOptions Options; + private ILogger _logger; + private int _count = 0; + private bool _forceReload = false; + private object _lock = new object(); + + public ConcurrentDictionary MachineCache = new ConcurrentDictionary(); + public ConcurrentDictionary> NetworkCache = new ConcurrentDictionary>(); + public ConcurrentDictionary DatastoreCache = new ConcurrentDictionary(); + public ConcurrentDictionary VmGuids = new ConcurrentDictionary(); + + public VsphereConnection(VsphereHost host, VsphereOptions options, ILogger logger) + { + Host = host; + Options = options; + _logger = logger; + } + + public async Task> Load() + { + var machineCache = Enumerable.Empty(); + + try + { + _logger.LogInformation($"Starting Connect Loop for {Host.Address} at {DateTime.UtcNow}"); + + if (!Host.Enabled) + { + _logger.LogInformation("Vsphere disabled, skipping"); + } + else + { + await Connect(); + + if (_count == Options.LoadCacheAfterIterations) + { + _count = 0; + } + + if (_count == 0 || _forceReload) + { + lock (_lock) + { + _forceReload = false; + _count = 0; + } + + machineCache = await LoadCache(); + } + + _logger.LogInformation($"Finished Connect Loop for {Host.Address} at {DateTime.UtcNow} with {MachineCache.Count()} Machines"); + _count++; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Exception encountered in ConnectionService loop"); + _count = 0; + } + + return machineCache; + } + + #region Connection Handling + + private async Task Connect() + { + // check whether session is expiring + if (Session != null && (DateTime.Compare(DateTime.UtcNow, Session.lastActiveTime.AddMinutes(Options.ConnectionRefreshIntervalMinutes)) >= 0)) + { + _logger.LogDebug("Connect(): Session is more than 20 minutes old"); + + // renew session because it expires at 30 minutes (maybe 120 minutes on newer vc) + _logger.LogInformation($"Connect(): renewing connection to {Host.Address}...[{Host.Username}]"); + try + { + var client = new VimPortTypeClient(VimPortTypeClient.EndpointConfiguration.VimPort, $"https://{Host.Address}/sdk"); + var sic = await client.RetrieveServiceContentAsync(new ManagedObjectReference { type = "ServiceInstance", Value = "ServiceInstance" }); + var props = sic.propertyCollector; + var session = await client.LoginAsync(sic.sessionManager, Host.Username, Host.Password, null); + + var oldClient = Client; + Client = client; + Sic = sic; + Props = props; + Session = session; + + await oldClient.CloseAsync(); + oldClient.Dispose(); + } + catch (Exception ex) + { + // no connection: Failed with Object reference not set to an instance of an object + _logger.LogError(0, ex, $"Connect(): Failed with " + ex.Message); + _logger.LogError(0, ex, $"Connect(): User: " + Host.Username); + Disconnect(); + } + } + + if (Client != null && Client.State == CommunicationState.Opened) + { + _logger.LogDebug("Connect(): CommunicationState.Opened"); + ServiceContent sic = Sic; + UserSession session = Session; + bool isNull = false; + + if (Sic == null) + { + sic = await ConnectToHost(Client); + isNull = true; + } + + if (Session == null) + { + session = await ConnectToSession(Client, sic); + isNull = true; + } + + if (isNull) + { + Session = session; + Props = sic.propertyCollector; + Sic = sic; + } + + try + { + var x = await Client.RetrieveServiceContentAsync(new ManagedObjectReference { type = "ServiceInstance", Value = "ServiceInstance" }); + return; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error checking vcenter connection. Disconnecting."); + Disconnect(); + } + } + + if (Client != null && Client.State == CommunicationState.Faulted) + { + _logger.LogDebug($"Connect(): https://{Host.Address}/sdk CommunicationState is Faulted."); + Disconnect(); + } + + if (Client == null) + { + try + { + _logger.LogDebug($"Connect(): Instantiating client https://{Host.Address}/sdk"); + var client = new VimPortTypeClient(VimPortTypeClient.EndpointConfiguration.VimPort, $"https://{Host.Address}/sdk"); + _logger.LogDebug($"Connect(): client: [{Client}]"); + + var sic = await ConnectToHost(client); + var session = await ConnectToSession(client, sic); + + Session = session; + Props = sic.propertyCollector; + Sic = sic; + Client = client; + } + catch (Exception ex) + { + _logger.LogError(0, ex, $"Connect(): Failed with " + ex.Message); + } + } + } + + private async Task ConnectToHost(VimPortTypeClient client) + { + _logger.LogInformation($"Connect(): Connecting to {Host.Address}..."); + var sic = await client.RetrieveServiceContentAsync(new ManagedObjectReference { type = "ServiceInstance", Value = "ServiceInstance" }); + return sic; + } + + private async Task ConnectToSession(VimPortTypeClient client, ServiceContent sic) + { + _logger.LogInformation($"Connect(): logging into {Host.Address}...[{Host.Username}]"); + var session = await client.LoginAsync(sic.sessionManager, Host.Username, Host.Password, null); + _logger.LogInformation($"Connect(): Session created."); + return session; + } + + public void Disconnect() + { + _logger.LogInformation($"Disconnect()"); + Client.Dispose(); + Client = null; + Sic = null; + Session = null; + } + + #endregion + + #region Cache Setup + + private async Task> LoadCache() + { + var plan = new TraversalSpec + { + name = "FolderTraverseSpec", + type = "Folder", + path = "childEntity", + selectSet = new SelectionSpec[] { + + new TraversalSpec() + { + type = "Datacenter", + path = "networkFolder", + selectSet = new SelectionSpec[] { + new SelectionSpec { + name = "FolderTraverseSpec" + } + } + }, + + new TraversalSpec() + { + type = "Datacenter", + path = "vmFolder", + selectSet = new SelectionSpec[] { + new SelectionSpec { + name = "FolderTraverseSpec" + } + } + }, + + new TraversalSpec() + { + type = "Datacenter", + path = "datastore", + selectSet = new SelectionSpec[] { + new SelectionSpec { + name = "FolderTraverseSpec" + } + } + }, + + new TraversalSpec() + { + type = "Folder", + path = "childEntity", + selectSet = new SelectionSpec[] { + new SelectionSpec { + name = "FolderTraverseSpec" + } + } + }, + } + }; + + var props = new PropertySpec[] + { + new PropertySpec + { + type = "DistributedVirtualSwitch", + pathSet = new string[] { "name", "uuid", "config.uplinkPortgroup" } + }, + + new PropertySpec + { + type = "DistributedVirtualPortgroup", + pathSet = new string[] { "name", "host", "config.distributedVirtualSwitch" } + }, + + new PropertySpec + { + type = "Network", + pathSet = new string[] { "name", "host" } + }, + + new PropertySpec + { + type = "VirtualMachine", + pathSet = new string[] { "name", "config.uuid", "summary.runtime.powerState", "guest.net" } + }, + + new PropertySpec + { + type = "Datastore", + pathSet = new string[] { "name", "browser" } + } + }; + + ObjectSpec objectspec = new ObjectSpec + { + obj = Sic.rootFolder, + selectSet = new SelectionSpec[] { plan } + }; + + PropertyFilterSpec filter = new PropertyFilterSpec + { + propSet = props, + objectSet = new ObjectSpec[] { objectspec } + }; + + PropertyFilterSpec[] filters = new PropertyFilterSpec[] { filter }; + + _logger.LogInformation($"Starting RetrieveProperties at {DateTime.UtcNow}"); + RetrievePropertiesResponse response = await Client.RetrievePropertiesAsync(Props, filters); + _logger.LogInformation($"Finished RetrieveProperties at {DateTime.UtcNow}"); + + _logger.LogInformation($"Starting LoadMachineCache at {DateTime.UtcNow}"); + var machineCache = LoadMachineCache(response.returnval.FindType("VirtualMachine")); + _logger.LogInformation($"Finished LoadMachineCache at {DateTime.UtcNow}"); + + _logger.LogInformation($"Starting LoadNetworkCache at {DateTime.UtcNow}"); + LoadNetworkCache( + response.returnval.FindType("DistributedVirtualSwitch"), + response.returnval.Where(o => o.obj.type.EndsWith("Network") || o.obj.type.EndsWith("DistributedVirtualPortgroup")).ToArray()); + _logger.LogInformation($"Finished LoadNetworkCache at {DateTime.UtcNow}"); + + _logger.LogInformation($"Starting LoadDatastoreCache at {DateTime.UtcNow}"); + LoadDatastoreCache(response.returnval.FindType("Datastore")); + _logger.LogInformation($"Finished LoadDatastoreCache at {DateTime.UtcNow}"); + + return machineCache; + } + + private IEnumerable LoadMachineCache(VimClient.ObjectContent[] virtualMachines) + { + IEnumerable existingMachineIds = MachineCache.Keys; + List currentMachineIds = new List(); + List vsphereVirtualMachines = new List(); + + foreach (var vm in virtualMachines) + { + string name = string.Empty; + + try + { + name = vm.GetProperty("name") as string; + + var idObj = vm.GetProperty("config.uuid"); + + if (idObj == null) + { + _logger.LogError($"Unable to load machine {name} - {vm.obj.Value}. Invalid UUID"); + continue; + } + + var toolsStatus = vm.GetProperty("summary.guest.toolsStatus") as Nullable; + VirtualMachineToolsStatus vmToolsStatus = VirtualMachineToolsStatus.toolsNotRunning; + if (toolsStatus != null) + { + vmToolsStatus = toolsStatus.Value; + } + + var guid = Guid.Parse(idObj as string); + var virtualMachine = new VsphereVirtualMachine + { + //HostReference = ((ManagedObjectReference)vm.GetProperty("summary.runtime.host")).Value, + Id = guid, + Name = name, + Reference = vm.obj, + State = (VirtualMachinePowerState)vm.GetProperty("summary.runtime.powerState") == VirtualMachinePowerState.poweredOn ? "on" : "off", + VmToolsStatus = vmToolsStatus, + IpAddresses = ((GuestNicInfo[])vm.GetProperty("guest.net")).Where(x => x.ipAddress != null).SelectMany(x => x.ipAddress).ToArray() + }; + + vsphereVirtualMachines.Add(virtualMachine); + + MachineCache.AddOrUpdate(virtualMachine.Id, virtualMachine.Reference, (k, v) => v = virtualMachine.Reference); + currentMachineIds.Add(virtualMachine.Id); + VmGuids.AddOrUpdate(vm.obj.Value, guid, (k, v) => (v = guid)); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error refreshing Virtual Machine {name} - {vm.obj.Value}"); + } + } + + foreach (Guid existingId in existingMachineIds.Except(currentMachineIds)) + { + if (MachineCache.TryRemove(existingId, out ManagedObjectReference stale)) + { + _logger.LogDebug($"removing stale cache entry {stale.Value}"); + } + } + + return vsphereVirtualMachines; + } + + private void LoadNetworkCache(VimClient.ObjectContent[] distributedSwitches, VimClient.ObjectContent[] networks) + { + Dictionary> networkCache = new Dictionary>(); + IEnumerable existingHosts = NetworkCache.Keys; + List currentHosts = new List(); + + foreach (var net in networks) + { + string name = null; + + try + { + name = net.GetProperty("name") as string; + Network network = null; + + if (net.obj.type == "Network") + { + network = new Network + { + IsDistributed = false, + Name = name, + SwitchId = null + }; + } + else if (net.obj.type == "DistributedVirtualPortgroup") + { + var dSwitchReference = net.GetProperty("config.distributedVirtualSwitch") as ManagedObjectReference; + var dSwitch = distributedSwitches.Where(x => x.obj.Value == dSwitchReference.Value).FirstOrDefault(); + + if (dSwitch != null) + { + var uplinkPortgroups = dSwitch.GetProperty("config.uplinkPortgroup") as ManagedObjectReference[]; + if (uplinkPortgroups.Select(x => x.Value).Contains(net.obj.Value)) + { + // Skip uplink portgroups + continue; + } + else + { + network = new Network + { + IsDistributed = true, + Name = name, + SwitchId = dSwitch.GetProperty("uuid") as string, + Reference = net.obj.Value + }; + } + } + } + else + { + _logger.LogError($"Unexpected type for Network {name}: {net.obj.type}"); + continue; + } + + if (network != null) + { + foreach (var host in net.GetProperty("host") as ManagedObjectReference[]) + { + string hostReference = host.Value; + + if (!networkCache.ContainsKey(hostReference)) + networkCache.Add(hostReference, new List()); + + networkCache[hostReference].Add(network); + + if (!currentHosts.Contains(hostReference)) + { + currentHosts.Add(hostReference); + } + } + } + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error refreshing Network {name} - {net.obj.Value}"); + } + } + + foreach (var kvp in networkCache) + { + NetworkCache.AddOrUpdate(kvp.Key, kvp.Value, (k, v) => (v = kvp.Value)); + } + + foreach (string existingHost in existingHosts.Except(currentHosts)) + { + if (NetworkCache.TryRemove(existingHost, out List stale)) + { + _logger.LogDebug($"removing stale network cache entry for Host {existingHost}"); + } + } + } + + private void LoadDatastoreCache(VimClient.ObjectContent[] rawDatastores) + { + IEnumerable cachedDatastoreNames = DatastoreCache.Keys; + List activeDatastoreNames = new List(); + Dictionary datastores = new Dictionary(); + foreach (var rawDatastore in rawDatastores) + { + try + { + Datastore datastore = new Datastore + { + Name = rawDatastore.GetProperty("name").ToString(), + Reference = rawDatastore.obj, + Browser = rawDatastore.GetProperty("browser") as ManagedObjectReference + }; + DatastoreCache.TryAdd(rawDatastore.GetProperty("name").ToString(), datastore); + activeDatastoreNames.Add(datastore.Name); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Error refreshing Datastore {rawDatastore.obj.Value}"); + } + } + + // clean cache of non-active datastores + foreach (var dsName in cachedDatastoreNames) + { + if (!activeDatastoreNames.Contains(dsName)) + { + _logger.LogDebug($"removing stale datastore cache entry {dsName}"); + DatastoreCache.Remove(dsName, out Datastore stale); + } + } + } + + #endregion +} diff --git a/src/Player.Vm.Api/Domain/Vsphere/Options/VsphereOptions.cs b/src/Player.Vm.Api/Domain/Vsphere/Options/VsphereOptions.cs index f6f78a8..d66d3f9 100644 --- a/src/Player.Vm.Api/Domain/Vsphere/Options/VsphereOptions.cs +++ b/src/Player.Vm.Api/Domain/Vsphere/Options/VsphereOptions.cs @@ -5,23 +5,24 @@ namespace Player.Vm.Api.Domain.Vsphere.Options { public class VsphereOptions { - public bool Enabled { get; set; } - public string Host { get; set; } - - public string Username { get; set; } - - public string Password { get; set; } - public int ConnectionRetryIntervalSeconds { get; set; } public int ConnectionRefreshIntervalMinutes { get; set; } public int LoadCacheAfterIterations { get; set; } public bool LogConsoleAccess { get; set; } - - public string DsName { get; set; } - public string BaseFolder { get; set; } - public int Timeout { get; set; } public int CheckTaskProgressIntervalMilliseconds { get; set; } public int ReCheckTaskProgressIntervalMilliseconds { get; set; } public int HealthAllowanceSeconds { get; set; } + + public VsphereHost[] Hosts { get; set; } + } + + public class VsphereHost + { + public bool Enabled { get; set; } = true; + public string Address { get; set; } + public string Username { get; set; } + public string Password { get; set; } + public string DsName { get; set; } + public string BaseFolder { get; set; } } } diff --git a/src/Player.Vm.Api/Domain/Vsphere/Services/ConnectionService.cs b/src/Player.Vm.Api/Domain/Vsphere/Services/ConnectionService.cs index 6710af6..9144487 100644 --- a/src/Player.Vm.Api/Domain/Vsphere/Services/ConnectionService.cs +++ b/src/Player.Vm.Api/Domain/Vsphere/Services/ConnectionService.cs @@ -8,12 +8,10 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.ServiceModel; using System.Threading; using System.Threading.Tasks; using VimClient; using Player.Vm.Api.Domain.Vsphere.Models; -using Player.Vm.Api.Domain.Vsphere.Extensions; using Player.Vm.Api.Domain.Vsphere.Options; using Microsoft.Extensions.DependencyInjection; using Player.Vm.Api.Data; @@ -23,654 +21,235 @@ using Player.Vm.Api.Infrastructure.Extensions; using Player.Vm.Api.Domain.Services.HealthChecks; -namespace Player.Vm.Api.Domain.Vsphere.Services +namespace Player.Vm.Api.Domain.Vsphere.Services; + +public interface IConnectionService +{ + ManagedObjectReference GetMachineById(Guid id); + Guid? GetVmIdByRef(string reference, string vsphereHost); + List GetNetworksByHost(string hostReference, string vsphereHost); + Network GetNetworkByReference(string networkReference, string vsphereHost); + Network GetNetworkByName(string networkName, string vsphereHost); + Datastore GetDatastoreByName(string dsName, string vsphereHost); + VsphereConnection GetConnection(string hostname); + VsphereAggregate GetAggregate(Guid id); + IEnumerable GetAllConnections(); +} + +public class ConnectionService : BackgroundService, IConnectionService { - public interface IConnectionService + private readonly ILogger _logger; + private readonly IOptionsMonitor _optionsMonitor; + private readonly IServiceProvider _serviceProvider; + + private AsyncAutoResetEvent _resetEvent = new AsyncAutoResetEvent(false); + private readonly ConnectionServiceHealthCheck _connectionServiceHealthCheck; + + public ConcurrentDictionary _connections = new ConcurrentDictionary(); // address to connection + public ConcurrentDictionary _machines = new ConcurrentDictionary(); // machine to vsphere address + + public ConnectionService( + IOptionsMonitor vsphereOptionsMonitor, + ILogger logger, + IServiceProvider serviceProvider, + ConnectionServiceHealthCheck connectionServiceHealthCheck + ) { - VimPortTypeClient GetClient(); - ServiceContent GetServiceContent(); - UserSession GetSession(); - ManagedObjectReference GetProps(); - ManagedObjectReference GetMachineById(Guid id); - Guid? GetVmIdByRef(string reference); - List GetNetworksByHost(string hostReference); - Network GetNetworkByReference(string networkReference); - Network GetNetworkByName(string networkName); - Datastore GetDatastoreByName(string dsName); - void ReloadCache(); + _optionsMonitor = vsphereOptionsMonitor; + _logger = logger; + _serviceProvider = serviceProvider; + _connectionServiceHealthCheck = connectionServiceHealthCheck; } - public class ConnectionService : BackgroundService, IConnectionService + protected override async Task ExecuteAsync(CancellationToken cancellationToken) { - private readonly ILogger _logger; - private VsphereOptions _options; - private readonly IOptionsMonitor _optionsMonitor; - private readonly IServiceProvider _serviceProvider; - - private VimPortTypeClient _client; - private ServiceContent _sic; - private UserSession _session; - private ManagedObjectReference _props; - - public ConcurrentDictionary _machineCache = new ConcurrentDictionary(); - public ConcurrentDictionary> _networkCache = new ConcurrentDictionary>(); - public ConcurrentDictionary _datastoreCache = new ConcurrentDictionary(); - public ConcurrentDictionary _vmGuids = new ConcurrentDictionary(); - - private object _lock = new object(); - private AsyncAutoResetEvent _resetEvent = new AsyncAutoResetEvent(false); - private bool _forceReload = false; - private readonly ConnectionServiceHealthCheck _connectionServiceHealthCheck; - - public ConnectionService( - IOptionsMonitor vsphereOptionsMonitor, - ILogger logger, - IServiceProvider serviceProvider, - ConnectionServiceHealthCheck connectionServiceHealthCheck - ) - { - _options = vsphereOptionsMonitor.CurrentValue; - _optionsMonitor = vsphereOptionsMonitor; - _logger = logger; - _serviceProvider = serviceProvider; - _connectionServiceHealthCheck = connectionServiceHealthCheck; - _connectionServiceHealthCheck.HealthAllowance = _options.HealthAllowanceSeconds; - } + await Task.Yield(); - public VimPortTypeClient GetClient() + while (!cancellationToken.IsCancellationRequested) { - return _client; - } + var taskDict = new Dictionary>>(); - public ServiceContent GetServiceContent() - { - return _sic; - } - - public UserSession GetSession() - { - return _session; - } - - public ManagedObjectReference GetProps() - { - return _props; - } - - public ManagedObjectReference GetMachineById(Guid id) - { - ManagedObjectReference machineReference = null; - _machineCache.TryGetValue(id, out machineReference); - return machineReference; - } - - public Guid? GetVmIdByRef(string reference) - { - Guid id; - if (_vmGuids.TryGetValue(reference, out id)) + foreach (var host in _optionsMonitor.CurrentValue.Hosts) { - return id; - } - else - { - return null; - } - } + // Create or update Host + var connection = _connections.GetOrAdd(host.Address, x => new VsphereConnection(host, _optionsMonitor.CurrentValue, _logger)); + connection.Options = _optionsMonitor.CurrentValue; + connection.Host = host; - public List GetNetworksByHost(string hostReference) - { - List networks; - _networkCache.TryGetValue(hostReference, out networks); - - if (networks == null) - networks = new List(); - - return networks; - } + taskDict.Add(connection.Address, connection.Load()); + } - public Network GetNetworkByReference(string networkReference) - { - return _networkCache.Values.SelectMany(x => x).Where(n => n.Reference == networkReference).FirstOrDefault(); - } + var result = await Task.WhenAll(taskDict.Values); + this.ProcessTasks(taskDict); - public Network GetNetworkByName(string networkName) - { - return _networkCache.Values.SelectMany(x => x).Where(n => n.Name == networkName).FirstOrDefault(); - } + await this.UpdateVms(result.SelectMany(x => x)); - public Datastore GetDatastoreByName(string dsName) - { - Datastore datastore = null; - _datastoreCache.TryGetValue(dsName, out datastore); - return datastore; + _connectionServiceHealthCheck.HealthAllowance = _optionsMonitor.CurrentValue.HealthAllowanceSeconds; + _connectionServiceHealthCheck.CompletedRun(); + await _resetEvent.WaitAsync(new TimeSpan(0, 0, _optionsMonitor.CurrentValue.ConnectionRetryIntervalSeconds)); } + } - protected override async Task ExecuteAsync(CancellationToken cancellationToken) + private void ProcessTasks(Dictionary>> taskDict) + { + // Add or update machines cache + foreach (var kvp in taskDict) { - await Task.Yield(); - - int count = 0; + var machines = kvp.Value.Result; - while (!cancellationToken.IsCancellationRequested) + foreach (var machine in machines) { - try - { - _logger.LogInformation($"Starting Connect Loop at {DateTime.UtcNow}"); - - _options = _optionsMonitor.CurrentValue; - - if (!_options.Enabled) - { - _logger.LogInformation("Vsphere disabled, skipping"); - } - else - { - await Connect(); - - if (count == _options.LoadCacheAfterIterations) - { - count = 0; - } - - if (count == 0 || _forceReload) - { - lock (_lock) - { - _forceReload = false; - count = 0; - } - - await LoadCache(); - } - - _logger.LogInformation($"Finished Connect Loop at {DateTime.UtcNow} with {_machineCache.Count()} Machines"); - count++; - } - } - catch (Exception ex) - { - _logger.LogError(ex, "Exception encountered in ConnectionService loop"); - count = 0; - } - - _connectionServiceHealthCheck.CompletedRun(); - await _resetEvent.WaitAsync(new TimeSpan(0, 0, _options.ConnectionRetryIntervalSeconds)); + _machines.AddOrUpdate(machine.Id, kvp.Key, (k, v) => v = kvp.Key); } } - public void ReloadCache() + // Remove machines that no longer exist from cache + var allMachines = _connections.Values.SelectMany(x => x.MachineCache.Select(y => y.Key)); + + foreach (var kvp in _machines) { - lock (_lock) + if (!allMachines.Contains(kvp.Key)) { - _forceReload = true; - _resetEvent.Set(); + _machines.TryRemove(kvp); } } + } - #region Cache Setup - - private async Task LoadCache() + private async Task UpdateVms(IEnumerable vsphereVirtualMachines) + { + using (var scope = _serviceProvider.CreateScope()) { - var plan = new TraversalSpec - { - name = "FolderTraverseSpec", - type = "Folder", - path = "childEntity", - selectSet = new SelectionSpec[] { - - new TraversalSpec() - { - type = "Datacenter", - path = "networkFolder", - selectSet = new SelectionSpec[] { - new SelectionSpec { - name = "FolderTraverseSpec" - } - } - }, - - new TraversalSpec() - { - type = "Datacenter", - path = "vmFolder", - selectSet = new SelectionSpec[] { - new SelectionSpec { - name = "FolderTraverseSpec" - } - } - }, - - new TraversalSpec() - { - type = "Datacenter", - path = "datastore", - selectSet = new SelectionSpec[] { - new SelectionSpec { - name = "FolderTraverseSpec" - } - } - }, - - new TraversalSpec() - { - type = "Folder", - path = "childEntity", - selectSet = new SelectionSpec[] { - new SelectionSpec { - name = "FolderTraverseSpec" - } - } - }, - } - }; + var dbContext = scope.ServiceProvider.GetRequiredService(); + var vms = await dbContext.Vms.ToArrayAsync(); - var props = new PropertySpec[] + foreach (var vsphereVirtualMachine in vsphereVirtualMachines) { - new PropertySpec - { - type = "DistributedVirtualSwitch", - pathSet = new string[] { "name", "uuid", "config.uplinkPortgroup" } - }, + var vm = vms.FirstOrDefault(x => x.Id == vsphereVirtualMachine.Id); - new PropertySpec + if (vm != null) { - type = "DistributedVirtualPortgroup", - pathSet = new string[] { "name", "host", "config.distributedVirtualSwitch" } - }, - - new PropertySpec - { - type = "Network", - pathSet = new string[] { "name", "host" } - }, - - new PropertySpec - { - type = "VirtualMachine", - pathSet = new string[] { "name", "config.uuid", "summary.runtime.powerState", "guest.net" } - }, - - new PropertySpec - { - type = "Datastore", - pathSet = new string[] { "name", "browser" } + var powerState = vsphereVirtualMachine.State == "on" ? PowerState.On : PowerState.Off; + vm.PowerState = powerState; + vm.IpAddresses = vsphereVirtualMachine.IpAddresses; + vm.Type = VmType.Vsphere; } - }; - - ObjectSpec objectspec = new ObjectSpec(); - objectspec.obj = _sic.rootFolder; - objectspec.selectSet = new SelectionSpec[] { plan }; - - PropertyFilterSpec filter = new PropertyFilterSpec(); - filter.propSet = props; - filter.objectSet = new ObjectSpec[] { objectspec }; - - PropertyFilterSpec[] filters = new PropertyFilterSpec[] { filter }; - - _logger.LogInformation($"Starting RetrieveProperties at {DateTime.UtcNow}"); - RetrievePropertiesResponse response = await _client.RetrievePropertiesAsync(_props, filters); - _logger.LogInformation($"Finished RetrieveProperties at {DateTime.UtcNow}"); - - _logger.LogInformation($"Starting LoadMachineCache at {DateTime.UtcNow}"); - await LoadMachineCache(response.returnval.FindType("VirtualMachine")); - _logger.LogInformation($"Finished LoadMachineCache at {DateTime.UtcNow}"); - - _logger.LogInformation($"Starting LoadNetworkCache at {DateTime.UtcNow}"); - LoadNetworkCache( - response.returnval.FindType("DistributedVirtualSwitch"), - response.returnval.Where(o => o.obj.type.EndsWith("Network") || o.obj.type.EndsWith("DistributedVirtualPortgroup")).ToArray()); - _logger.LogInformation($"Finished LoadNetworkCache at {DateTime.UtcNow}"); + } - _logger.LogInformation($"Starting LoadDatastoreCache at {DateTime.UtcNow}"); - LoadDatastoreCache(response.returnval.FindType("Datastore")); - _logger.LogInformation($"Finished LoadDatastoreCache at {DateTime.UtcNow}"); + var count = await dbContext.SaveChangesAsync(); } + } - private async Task LoadMachineCache(VimClient.ObjectContent[] virtualMachines) - { - IEnumerable existingMachineIds = _machineCache.Keys; - List currentMachineIds = new List(); - Dictionary vsphereVirtualMachines = new Dictionary(); - - foreach (var vm in virtualMachines) - { - string name = string.Empty; + public VsphereAggregate GetAggregate(Guid id) + { + string address; - try - { - name = vm.GetProperty("name") as string; - - var idObj = vm.GetProperty("config.uuid"); - - if (idObj == null) - { - _logger.LogError($"Unable to load machine {name} - {vm.obj.Value}. Invalid UUID"); - continue; - } - - var toolsStatus = vm.GetProperty("summary.guest.toolsStatus") as Nullable; - VirtualMachineToolsStatus vmToolsStatus = VirtualMachineToolsStatus.toolsNotRunning; - if (toolsStatus != null) - { - vmToolsStatus = toolsStatus.Value; - } - - var guid = Guid.Parse(idObj as string); - var virtualMachine = new VsphereVirtualMachine - { - //HostReference = ((ManagedObjectReference)vm.GetProperty("summary.runtime.host")).Value, - Id = guid, - Name = name, - Reference = vm.obj, - State = (VirtualMachinePowerState)vm.GetProperty("summary.runtime.powerState") == VirtualMachinePowerState.poweredOn ? "on" : "off", - VmToolsStatus = vmToolsStatus, - IpAddresses = ((GuestNicInfo[])vm.GetProperty("guest.net")).Where(x => x.ipAddress != null).SelectMany(x => x.ipAddress).ToArray() - }; - - vsphereVirtualMachines.Add(virtualMachine.Id, virtualMachine); - - _machineCache.AddOrUpdate(virtualMachine.Id, virtualMachine.Reference, (k, v) => (v = virtualMachine.Reference)); - currentMachineIds.Add(virtualMachine.Id); - _vmGuids.AddOrUpdate(vm.obj.Value, guid, (k, v) => (v = guid)); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error refreshing Virtual Machine {name} - {vm.obj.Value}"); - } - } + if (_machines.TryGetValue(id, out address)) + { + VsphereConnection connection; - foreach (Guid existingId in existingMachineIds.Except(currentMachineIds)) + if (_connections.TryGetValue(address, out connection)) { - if (_machineCache.TryRemove(existingId, out ManagedObjectReference stale)) + ManagedObjectReference machineReference; + if (connection.MachineCache.TryGetValue(id, out machineReference)) { - _logger.LogDebug($"removing stale cache entry {stale.Value}"); + return new VsphereAggregate(connection, machineReference); } } - - await UpdateVms(vsphereVirtualMachines); } - private async Task UpdateVms(Dictionary vsphereVirtualMachines) - { - using (var scope = _serviceProvider.CreateScope()) - { - var dbContext = scope.ServiceProvider.GetRequiredService(); - var vms = await dbContext.Vms.ToArrayAsync(); + return null; + } - foreach (var vm in vms) - { - VsphereVirtualMachine vsphereVirtualMachine; - - if (vsphereVirtualMachines.TryGetValue(vm.Id, out vsphereVirtualMachine)) - { - var powerState = vsphereVirtualMachine.State == "on" ? PowerState.On : PowerState.Off; - vm.PowerState = powerState; - vm.IpAddresses = vsphereVirtualMachine.IpAddresses; - vm.Type = VmType.Vsphere; - } - } + public IEnumerable GetAllConnections() + { + return _connections.Values; + } - var count = await dbContext.SaveChangesAsync(); - } - } + public VsphereConnection GetConnection(string hostname) + { + VsphereConnection connection = null; + _connections.TryGetValue(hostname, out connection); + return connection; + } - private void LoadNetworkCache(VimClient.ObjectContent[] distributedSwitches, VimClient.ObjectContent[] networks) + public ManagedObjectReference GetMachineById(Guid id) + { + ManagedObjectReference machineReference; + foreach (var connection in _connections.Values) { - Dictionary> networkCache = new Dictionary>(); - IEnumerable existingHosts = _networkCache.Keys; - List currentHosts = new List(); - - foreach (var net in networks) + if (connection.MachineCache.TryGetValue(id, out machineReference)) { - string name = null; - - try - { - name = net.GetProperty("name") as string; - Network network = null; - - if (net.obj.type == "Network") - { - network = new Network - { - IsDistributed = false, - Name = name, - SwitchId = null - }; - } - else if (net.obj.type == "DistributedVirtualPortgroup") - { - var dSwitchReference = net.GetProperty("config.distributedVirtualSwitch") as ManagedObjectReference; - var dSwitch = distributedSwitches.Where(x => x.obj.Value == dSwitchReference.Value).FirstOrDefault(); - - if (dSwitch != null) - { - var uplinkPortgroups = dSwitch.GetProperty("config.uplinkPortgroup") as ManagedObjectReference[]; - if (uplinkPortgroups.Select(x => x.Value).Contains(net.obj.Value)) - { - // Skip uplink portgroups - continue; - } - else - { - network = new Network - { - IsDistributed = true, - Name = name, - SwitchId = dSwitch.GetProperty("uuid") as string, - Reference = net.obj.Value - }; - } - } - } - else - { - _logger.LogError($"Unexpected type for Network {name}: {net.obj.type}"); - continue; - } - - if (network != null) - { - foreach (var host in net.GetProperty("host") as ManagedObjectReference[]) - { - string hostReference = host.Value; - - if (!networkCache.ContainsKey(hostReference)) - networkCache.Add(hostReference, new List()); - - networkCache[hostReference].Add(network); - - if (!currentHosts.Contains(hostReference)) - { - currentHosts.Add(hostReference); - } - } - } - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error refreshing Network {name} - {net.obj.Value}"); - } - } - - foreach (var kvp in networkCache) - { - _networkCache.AddOrUpdate(kvp.Key, kvp.Value, (k, v) => (v = kvp.Value)); - } - - foreach (string existingHost in existingHosts.Except(currentHosts)) - { - if (_networkCache.TryRemove(existingHost, out List stale)) - { - _logger.LogDebug($"removing stale network cache entry for Host {existingHost}"); - } + return machineReference; } } - private void LoadDatastoreCache(VimClient.ObjectContent[] rawDatastores) - { - IEnumerable cachedDatastoreNames = _datastoreCache.Keys; - List activeDatastoreNames = new List(); - Dictionary datastores = new Dictionary(); - foreach (var rawDatastore in rawDatastores) - { - try - { - Datastore datastore = new Datastore - { - Name = rawDatastore.GetProperty("name").ToString(), - Reference = rawDatastore.obj, - Browser = rawDatastore.GetProperty("browser") as ManagedObjectReference - }; - _datastoreCache.TryAdd(rawDatastore.GetProperty("name").ToString(), datastore); - activeDatastoreNames.Add(datastore.Name); - } - catch (Exception ex) - { - _logger.LogError(ex, $"Error refreshing Datastore {rawDatastore.obj.Value}"); - } - } + return null; + } - // clean cache of non-active datastores - foreach (var dsName in cachedDatastoreNames) + public Guid? GetVmIdByRef(string reference, string vsphereHost) + { + VsphereConnection connection; + if (_connections.TryGetValue(vsphereHost, out connection)) + { + Guid id; + if (connection.VmGuids.TryGetValue(reference, out id)) { - if (!activeDatastoreNames.Contains(dsName)) - { - _logger.LogDebug($"removing stale datastore cache entry {dsName}"); - _datastoreCache.Remove(dsName, out Datastore stale); - } + return id; } } - #endregion + return null; + } - #region Connection Handling + public List GetNetworksByHost(string hostReference, string vsphereHost) + { + VsphereConnection connection; + List networks = new List(); - private async Task Connect() + if (_connections.TryGetValue(vsphereHost, out connection)) { - // check whether session is expiring - if (_session != null && (DateTime.Compare(DateTime.UtcNow, _session.lastActiveTime.AddMinutes(_options.ConnectionRefreshIntervalMinutes)) >= 0)) - { - _logger.LogDebug("Connect(): Session is more than 20 minutes old"); - - // renew session because it expires at 30 minutes (maybe 120 minutes on newer vc) - _logger.LogInformation($"Connect(): renewing connection to {_options.Host}...[{_options.Username}]"); - try - { - var client = new VimPortTypeClient(VimPortTypeClient.EndpointConfiguration.VimPort, $"https://{_options.Host}/sdk"); - var sic = await client.RetrieveServiceContentAsync(new ManagedObjectReference { type = "ServiceInstance", Value = "ServiceInstance" }); - var props = sic.propertyCollector; - var session = await client.LoginAsync(sic.sessionManager, _options.Username, _options.Password, null); - - var oldClient = _client; - _client = client; - _sic = sic; - _props = props; - _session = session; - - await oldClient.CloseAsync(); - oldClient.Dispose(); - } - catch (Exception ex) - { - // no connection: Failed with Object reference not set to an instance of an object - _logger.LogError(0, ex, $"Connect(): Failed with " + ex.Message); - _logger.LogError(0, ex, $"Connect(): User: " + _options.Username); - Disconnect(); - } - } - - if (_client != null && _client.State == CommunicationState.Opened) - { - _logger.LogDebug("Connect(): CommunicationState.Opened"); - ServiceContent sic = _sic; - UserSession session = _session; - bool isNull = false; - - if (_sic == null) - { - sic = await ConnectToHost(_client); - isNull = true; - } - - if (_session == null) - { - session = await ConnectToSession(_client, sic); - isNull = true; - } - - if (isNull) - { - _session = session; - _props = sic.propertyCollector; - _sic = sic; - } + connection.NetworkCache.TryGetValue(hostReference, out networks); + } - try - { - var x = await _client.RetrieveServiceContentAsync(new ManagedObjectReference { type = "ServiceInstance", Value = "ServiceInstance" }); - return; - } - catch (Exception ex) - { - _logger.LogError(ex, "Error checking vcenter connection. Disconnecting."); - Disconnect(); - } - } + return networks; + } - if (_client != null && _client.State == CommunicationState.Faulted) - { - _logger.LogDebug($"Connect(): https://{_options.Host}/sdk CommunicationState is Faulted."); - Disconnect(); - } + public Network GetNetworkByReference(string networkReference, string vsphereHost) + { + VsphereConnection connection; + Network network = null; - if (_client == null) - { - try - { - _logger.LogDebug($"Connect(): Instantiating client https://{_options.Host}/sdk"); - var client = new VimPortTypeClient(VimPortTypeClient.EndpointConfiguration.VimPort, $"https://{_options.Host}/sdk"); - _logger.LogDebug($"Connect(): client: [{_client}]"); + if (_connections.TryGetValue(vsphereHost, out connection)) + { + network = connection.NetworkCache.Values.SelectMany(x => x).Where(n => n.Reference == networkReference).FirstOrDefault(); + } - var sic = await ConnectToHost(client); - var session = await ConnectToSession(client, sic); + return network; + } - _session = session; - _props = sic.propertyCollector; - _sic = sic; - _client = client; - } - catch (Exception ex) - { - _logger.LogError(0, ex, $"Connect(): Failed with " + ex.Message); - } - } - } + public Network GetNetworkByName(string networkName, string vsphereHost) + { + VsphereConnection connection; + Network network = null; - private async Task ConnectToHost(VimPortTypeClient client) + if (_connections.TryGetValue(vsphereHost, out connection)) { - _logger.LogInformation($"Connect(): Connecting to {_options.Host}..."); - var sic = await client.RetrieveServiceContentAsync(new ManagedObjectReference { type = "ServiceInstance", Value = "ServiceInstance" }); - return sic; + network = connection.NetworkCache.Values.SelectMany(x => x).Where(n => n.Name == networkName).FirstOrDefault(); } - private async Task ConnectToSession(VimPortTypeClient client, ServiceContent sic) - { - _logger.LogInformation($"Connect(): logging into {_options.Host}...[{_options.Username}]"); - var session = await client.LoginAsync(sic.sessionManager, _options.Username, _options.Password, null); - _logger.LogInformation($"Connect(): Session created."); - return session; - } + return network; + } - public void Disconnect() + public Datastore GetDatastoreByName(string dsName, string vsphereHost) + { + VsphereConnection connection; + Datastore datastore = null; + + if (_connections.TryGetValue(vsphereHost, out connection)) { - _logger.LogInformation($"Disconnect()"); - _client.Dispose(); - _client = null; - _sic = null; - _session = null; + connection.DatastoreCache.TryGetValue(dsName, out datastore); } - #endregion + return datastore; } -} +} \ No newline at end of file diff --git a/src/Player.Vm.Api/Domain/Vsphere/Services/MachineStateService.cs b/src/Player.Vm.Api/Domain/Vsphere/Services/MachineStateService.cs index 3ac1b42..a8733b4 100644 --- a/src/Player.Vm.Api/Domain/Vsphere/Services/MachineStateService.cs +++ b/src/Player.Vm.Api/Domain/Vsphere/Services/MachineStateService.cs @@ -18,6 +18,8 @@ using Player.Vm.Api.Domain.Models; using Nito.AsyncEx; using Player.Vm.Api.Infrastructure.Extensions; +using System.Collections.Concurrent; +using Player.Vm.Api.Domain.Vsphere.Models; namespace Player.Vm.Api.Domain.Vsphere.Services { @@ -29,19 +31,17 @@ public interface IMachineStateService public class MachineStateService : BackgroundService, IMachineStateService { private readonly ILogger _logger; - private VsphereOptions _options; private readonly IOptionsMonitor _optionsMonitor; private readonly IServiceProvider _serviceProvider; private VmContext _dbContext; private IVsphereService _vsphereService; private readonly IConnectionService _connectionService; private AsyncAutoResetEvent _resetEvent = new AsyncAutoResetEvent(false); - private DateTime _lastCheckedTime = DateTime.UtcNow; + private ConcurrentDictionary _lastCheckedTimes = new ConcurrentDictionary(); public MachineStateService( IOptionsMonitor optionsMonitor, ILogger logger, - IMemoryCache cache, IConnectionService connectionService, IServiceProvider serviceProvider ) @@ -69,11 +69,8 @@ protected override async Task ExecuteAsync(CancellationToken cancellationToken) { InitScope(scope); - if (_options.Enabled) - { - var events = await GetEvents(); - await ProcessEvents(events); - } + var events = await GetEvents(); + await ProcessEvents(events); } } catch (Exception ex) @@ -82,7 +79,7 @@ protected override async Task ExecuteAsync(CancellationToken cancellationToken) } await _resetEvent.WaitAsync( - new TimeSpan(0, 0, 0, 0, _options.CheckTaskProgressIntervalMilliseconds), + new TimeSpan(0, 0, 0, 0, _optionsMonitor.CurrentValue.CheckTaskProgressIntervalMilliseconds), cancellationToken); } } @@ -91,17 +88,52 @@ private void InitScope(IServiceScope scope) { _dbContext = scope.ServiceProvider.GetRequiredService(); _vsphereService = scope.ServiceProvider.GetRequiredService(); - _options = _optionsMonitor.CurrentValue; } - private async Task> GetEvents() + private async Task>> GetEvents() { - var now = DateTime.UtcNow; - var events = await _vsphereService.GetEvents(GetFilterSpec(_lastCheckedTime)); - _lastCheckedTime = now; + var connections = _connectionService.GetAllConnections(); + var taskList = new List>>>(); + var events = new Dictionary>(); + + foreach (var connection in connections) + { + if (connection.Enabled) + { + taskList.Add(GetEvents(connection)); + } + } + + var results = await Task.WhenAll(taskList); + + foreach (var kvp in results) + { + events.Add(kvp.Key, kvp.Value); + } + return events; } + private async Task>> GetEvents(VsphereConnection connection) + { + var lastCheckedTime = _lastCheckedTimes.GetOrAdd(connection.Address, DateTime.UtcNow); + IEnumerable events; + var now = DateTime.UtcNow; + + try + { + events = await _vsphereService.GetEvents(GetFilterSpec(lastCheckedTime), connection); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Exception getting events from {connection.Address}"); + return new KeyValuePair>(connection.Address, new List()); + } + + _lastCheckedTimes[connection.Address] = now; + return new KeyValuePair>(connection.Address, events); + } + private EventFilterSpec GetFilterSpec(DateTime beginTime) { var filterSpec = new EventFilterSpec() @@ -122,26 +154,30 @@ private EventFilterSpec GetFilterSpec(DateTime beginTime) return filterSpec; } - private async Task ProcessEvents(IEnumerable events) + private async Task ProcessEvents(Dictionary> events) { var eventDict = new Dictionary(); - if (!events.Any()) + if (!events.SelectMany(x => x.Value).Any()) { return; } - var filteredEvents = events.GroupBy(x => x.vm.vm.Value) + foreach (var eventList in events) + { + var connectionEvents = eventList.Value; + var filteredEvents = connectionEvents.GroupBy(x => x.vm.vm.Value) .Select(g => g.OrderByDescending(l => l.createdTime).First()) .ToArray(); - foreach (var evt in filteredEvents) - { - var id = _connectionService.GetVmIdByRef(evt.vm.vm.Value); - - if (id.HasValue) + foreach (var evt in filteredEvents) { - eventDict.TryAdd(id.Value, evt); + var id = _connectionService.GetVmIdByRef(evt.vm.vm.Value, eventList.Key); + + if (id.HasValue) + { + eventDict.TryAdd(id.Value, evt); + } } } diff --git a/src/Player.Vm.Api/Domain/Vsphere/Services/TaskService.cs b/src/Player.Vm.Api/Domain/Vsphere/Services/TaskService.cs index a457efb..c125a89 100644 --- a/src/Player.Vm.Api/Domain/Vsphere/Services/TaskService.cs +++ b/src/Player.Vm.Api/Domain/Vsphere/Services/TaskService.cs @@ -34,16 +34,12 @@ public class TaskService : BackgroundService, ITaskService { private readonly IHubContext _progressHub; private readonly ILogger _logger; - private VsphereOptions _options; private readonly IOptionsMonitor _optionsMonitor; private readonly IServiceProvider _serviceProvider; private VmContext _dbContext; private IConnectionService _connectionService; private IMachineStateService _machineStateService; - private VimPortTypeClient _client; - private ServiceContent _sic; - private ManagedObjectReference _props; private ConcurrentDictionary> _runningTasks = new ConcurrentDictionary>(); private AsyncAutoResetEvent _resetEvent = new AsyncAutoResetEvent(false); private bool _tasksPending = false; @@ -67,7 +63,6 @@ TaskServiceHealthCheck taskServiceHealthCheck _serviceProvider = serviceProvider; _machineStateService = machineStateService; _taskServiceHealthCheck = taskServiceHealthCheck; - _taskServiceHealthCheck.HealthAllowance = _optionsMonitor.CurrentValue.HealthAllowanceSeconds; } protected override async Task ExecuteAsync(CancellationToken cancellationToken) @@ -78,20 +73,12 @@ protected override async Task ExecuteAsync(CancellationToken cancellationToken) { try { - _options = _optionsMonitor.CurrentValue; - _client = _connectionService.GetClient(); - _sic = _connectionService.GetServiceContent(); - _props = _connectionService.GetProps(); _tasksPending = false; using (var scope = _serviceProvider.CreateScope()) { _dbContext = scope.ServiceProvider.GetRequiredService(); - - if (_options.Enabled) - { - await processTasks(); - } + await processTasks(); } } catch (Exception ex) @@ -100,9 +87,10 @@ protected override async Task ExecuteAsync(CancellationToken cancellationToken) } var intervalMilliseconds = _tasksPending ? - _options.ReCheckTaskProgressIntervalMilliseconds : - _options.CheckTaskProgressIntervalMilliseconds; + _optionsMonitor.CurrentValue.ReCheckTaskProgressIntervalMilliseconds : + _optionsMonitor.CurrentValue.CheckTaskProgressIntervalMilliseconds; + _taskServiceHealthCheck.HealthAllowance = _optionsMonitor.CurrentValue.HealthAllowanceSeconds; _taskServiceHealthCheck.CompletedRun(); await _resetEvent.WaitAsync(new TimeSpan(0, 0, 0, 0, intervalMilliseconds)); } @@ -120,32 +108,58 @@ private async Task processTasks() { try { - await this._progressHub.Clients.Group(vmTasks.Key).SendAsync("Progress", vmTasks.Value); + await _progressHub.Clients.Group(vmTasks.Key).SendAsync("Progress", vmTasks.Value); } catch (Exception ex) { - this._logger.LogError(ex, "Exception in processTasks"); + _logger.LogError(ex, "Exception in processTasks"); } } } private async Task getRecentTasks() { - if (_sic != null && _sic.taskManager != null) + var pendingVms = await _dbContext.Vms + .Include(x => x.VmTeams) + .Where(x => x.HasPendingTasks) + .ToArrayAsync(); + + var stillPendingVmIds = new List(); + + var responseDict = new Dictionary>(); + var connections = _connectionService.GetAllConnections(); + + foreach (var connection in connections) { - var pendingVms = await _dbContext.Vms - .Include(x => x.VmTeams) - .Where(x => x.HasPendingTasks) - .ToArrayAsync(); + if (connection.Enabled && connection.Sic != null && connection.Props != null) + { + PropertyFilterSpec[] filters = createPFSForRecentTasks(connection.Sic.taskManager); + var task = connection.Client.RetrievePropertiesAsync(connection.Props, filters); + responseDict.Add(connection, task); + } + } - var stillPendingVmIds = new List(); + try + { + await Task.WhenAll(responseDict.Select(x => x.Value)); + } + catch (Exception) + { + // Will check for exceptions in individual tasks below + } - PropertyFilterSpec[] filters = createPFSForRecentTasks(_sic.taskManager); - RetrievePropertiesResponse response = await _client.RetrievePropertiesAsync(_props, filters); - _runningTasks.Clear(); - var forceCheckMachineState = false; + _runningTasks.Clear(); + var forceCheckMachineState = false; - foreach (var task in response.returnval) + foreach (var kvp in responseDict) + { + if (kvp.Value.Status != TaskStatus.RanToCompletion) + { + _logger.LogError(kvp.Value.Exception, $"Exception retrieving tasks from {kvp.Key.Address}"); + continue; + } + + foreach (var task in kvp.Value.Result.returnval) { try { @@ -154,7 +168,7 @@ private async Task getRecentTasks() if (vmRef != null) { - vmId = _connectionService.GetVmIdByRef(vmRef); + vmId = _connectionService.GetVmIdByRef(vmRef, kvp.Key.Address); } var broadcastTime = DateTime.UtcNow.ToString(); @@ -181,7 +195,7 @@ private async Task getRecentTasks() } - if ((state == TaskInfoState.queued.ToString() || state == TaskInfoState.running.ToString())) + if (state == TaskInfoState.queued.ToString() || state == TaskInfoState.running.ToString()) { _tasksPending = true; @@ -192,7 +206,7 @@ private async Task getRecentTasks() } if (state == TaskInfoState.success.ToString() && - (this.GetPowerTaskTypes().Contains(taskType))) + this.GetPowerTaskTypes().Contains(taskType)) { if (vmId.HasValue) { @@ -202,34 +216,34 @@ private async Task getRecentTasks() } catch (Exception ex) { - this._logger.LogError(ex, "Exception processing task"); + _logger.LogError(ex, $"Exception processing task from {kvp.Key.Address}"); } } + } - foreach (var vm in pendingVms) + foreach (var vm in pendingVms) + { + if (!stillPendingVmIds.Contains(vm.Id)) { - if (!stillPendingVmIds.Contains(vm.Id)) - { - vm.HasPendingTasks = false; - } + vm.HasPendingTasks = false; } + } - var vmsToUpdate = await _dbContext.Vms - .Include(x => x.VmTeams) - .Where(x => stillPendingVmIds.Contains(x.Id)) - .ToArrayAsync(); + var vmsToUpdate = await _dbContext.Vms + .Include(x => x.VmTeams) + .Where(x => stillPendingVmIds.Contains(x.Id)) + .ToArrayAsync(); - foreach (var vm in vmsToUpdate) - { - vm.HasPendingTasks = true; - } + foreach (var vm in vmsToUpdate) + { + vm.HasPendingTasks = true; + } - await _dbContext.SaveChangesAsync(); + await _dbContext.SaveChangesAsync(); - if (forceCheckMachineState) - { - _machineStateService.CheckState(); - } + if (forceCheckMachineState) + { + _machineStateService.CheckState(); } } @@ -245,10 +259,11 @@ private string[] GetPowerTaskTypes() private PropertyFilterSpec[] createPFSForRecentTasks(ManagedObjectReference taskManagerRef) { - PropertySpec pSpec = new PropertySpec(); - pSpec.all = false; - pSpec.type = "Task"; - pSpec.pathSet = new String[] + PropertySpec pSpec = new PropertySpec + { + all = false, + type = "Task", + pathSet = new string[] { "info.entity", "info.descriptionId", @@ -258,24 +273,31 @@ private PropertyFilterSpec[] createPFSForRecentTasks(ManagedObjectReference task "info.cancelled", "info.error", "info.key" - }; + } + }; - ObjectSpec oSpec = new ObjectSpec(); - oSpec.obj = taskManagerRef; - oSpec.skip = false; - oSpec.skipSpecified = true; + ObjectSpec oSpec = new ObjectSpec + { + obj = taskManagerRef, + skip = false, + skipSpecified = true + }; - TraversalSpec tSpec = new TraversalSpec(); - tSpec.type = "TaskManager"; - tSpec.path = "recentTask"; - tSpec.skip = false; + TraversalSpec tSpec = new TraversalSpec + { + type = "TaskManager", + path = "recentTask", + skip = false + }; oSpec.selectSet = new SelectionSpec[] { tSpec }; - PropertyFilterSpec pfSpec = new PropertyFilterSpec(); - pfSpec.propSet = new PropertySpec[] { pSpec }; - pfSpec.objectSet = new ObjectSpec[] { oSpec }; + PropertyFilterSpec pfSpec = new PropertyFilterSpec + { + propSet = new PropertySpec[] { pSpec }, + objectSet = new ObjectSpec[] { oSpec } + }; return new PropertyFilterSpec[] { pfSpec }; } diff --git a/src/Player.Vm.Api/Domain/Vsphere/Services/VsphereService.cs b/src/Player.Vm.Api/Domain/Vsphere/Services/VsphereService.cs index 1f1ff12..04359c7 100644 --- a/src/Player.Vm.Api/Domain/Vsphere/Services/VsphereService.cs +++ b/src/Player.Vm.Api/Domain/Vsphere/Services/VsphereService.cs @@ -31,33 +31,27 @@ public interface IVsphereService Task ReconfigureVm(Guid id, Feature feature, string label, string newvalue); Task GetVmToolsStatus(Guid id); Task UploadFileToVm(Guid id, string username, string password, string filepath, Stream fileStream); - Task> GetIsos(string viewId, string subFolder); + Task> GetIsos(Guid vmId, string viewId, string subFolder); Task SetResolution(Guid id, int width, int height); Task BulkPowerOperation(Guid[] ids, PowerOperation operation); Task> BulkShutdown(Guid[] ids); Task> BulkReboot(Guid[] ids); Task> GetPowerState(IEnumerable machineIds); - Task> GetEvents(EventFilterSpec filterSpec); + Task> GetEvents(EventFilterSpec filterSpec, VsphereConnection connection); } public class VsphereService : IVsphereService { - private VsphereOptions _options; private RewriteHostOptions _rewriteHostOptions; private readonly ILogger _logger; int _pollInterval = 1000; - List _networks; - private VimPortTypeClient _client; - private ServiceContent _sic; - ManagedObjectReference _props; private readonly IConfiguration _configuration; private readonly IConnectionService _connectionService; private readonly IMapper _mapper; public VsphereService( - VsphereOptions options, IOptions rewriteHostOptions, ILogger logger, IConfiguration configuration, @@ -65,14 +59,9 @@ public VsphereService( IMapper mapper ) { - _options = options; _rewriteHostOptions = rewriteHostOptions.Value; - _logger = logger; _connectionService = connectionService; - _client = connectionService.GetClient(); - _sic = connectionService.GetServiceContent(); - _props = connectionService.GetProps(); _configuration = configuration; _mapper = mapper; } @@ -84,17 +73,19 @@ public async Task GetConsoleUrl(VsphereVirtualMachine machine) return null; } - return await GetConsoleUrl(machine.Id, machine.Reference); + return await GetConsoleUrl(machine.Id); } - public async Task GetConsoleUrl(Guid id, ManagedObjectReference vmReference) + public async Task GetConsoleUrl(Guid id) { + var aggregate = await this.GetVm(id); + VirtualMachineTicket ticket = null; string url = null; try { - ticket = await _client.AcquireTicketAsync(vmReference, "webmks"); + ticket = await aggregate.Connection.Client.AcquireTicketAsync(aggregate.MachineReference, "webmks"); } catch (Exception ex) { @@ -108,7 +99,7 @@ public async Task GetConsoleUrl(Guid id, ManagedObjectReference vmRefere if (ticket.host != null) host = ticket.host; else - host = _options.Host; + host = aggregate.Connection.Address; if (_rewriteHostOptions.RewriteHost) { @@ -124,48 +115,46 @@ public async Task GetConsoleUrl(Guid id, ManagedObjectReference vmRefere return url; } - //string uuid = "50303a87-2f5b-6d13-2211-a33556ba6e7f"; - public async Task GetConsoleUrl(Guid uuid) + public async Task GetVm(Guid id) { - ManagedObjectReference vmReference = null; + VsphereAggregate aggregate = _connectionService.GetAggregate(id); - _logger.LogDebug($"Aquiring webmks ticket for vm {uuid}"); + // Vm not found, check all connections for it + if (aggregate == null) + { + List> taskList = new List>(); - vmReference = await GetVm(uuid); + foreach (var connection in _connectionService.GetAllConnections()) + { + taskList.Add(this.FindVm(id, connection)); + } - if (vmReference == null) - return null; + var results = await Task.WhenAll(taskList); + aggregate = results.Where(x => x != null).FirstOrDefault(); + } - return await GetConsoleUrl(uuid, vmReference); + return aggregate; } - public async Task GetVm(Guid id) + private async Task FindVm(Guid id, VsphereConnection connection) { - ManagedObjectReference vmReference = _connectionService.GetMachineById(id); + VsphereAggregate aggregate = null; - if (vmReference == null && _client != null && _sic != null) + try { - try - { - vmReference = await _client.FindByUuidAsync(_sic.searchIndex, null, id.ToString(), true, false); - } - catch (Exception ex) + var vmReference = await connection.Client.FindByUuidAsync(connection.Sic.searchIndex, null, id.ToString(), true, false); + + if (vmReference != null) { - _logger.LogError(0, ex, $"Failed to get reference for " + id); - if (_client == null) - { - _logger.LogError(0, ex, $"_client is null"); - } - if (_sic == null) - { - _logger.LogError(0, ex, $"_sic is null"); - } - _logger.LogError(0, ex, $"Failed with " + ex.Message); - // should determine the cause + aggregate = new VsphereAggregate(connection, vmReference); } } + catch (Exception ex) + { + _logger.LogDebug(ex, $"Did not find machine with id: {id} on connection to {connection.Address}"); + } - return vmReference; + return aggregate; } public async Task PowerOnVm(Guid id) @@ -176,7 +165,8 @@ public async Task PowerOnVm(Guid id) ManagedObjectReference task; string state = null; - vmReference = await GetVm(id); + var aggregate = await this.GetVm(id); + vmReference = aggregate.MachineReference; if (vmReference == null) { @@ -194,7 +184,7 @@ public async Task PowerOnVm(Guid id) try { - task = await _client.PowerOnVM_TaskAsync(vmReference, null); + task = await aggregate.Connection.Client.PowerOnVM_TaskAsync(vmReference, null); // TaskInfo info = await WaitForVimTask(task); // if (info.state == TaskInfoState.success) { @@ -226,7 +216,8 @@ public async Task PowerOffVm(Guid id) ManagedObjectReference task; string state = null; - vmReference = await GetVm(id); + var aggregate = await this.GetVm(id); + vmReference = aggregate.MachineReference; if (vmReference == null) { @@ -247,7 +238,7 @@ public async Task PowerOffVm(Guid id) try { - task = await _client.PowerOffVM_TaskAsync(vmReference); + task = await aggregate.Connection.Client.PowerOffVM_TaskAsync(vmReference); //TaskInfo info = await WaitForVimTask(task); // if (info.state == TaskInfoState.success) { @@ -277,7 +268,8 @@ public async Task BulkPowerOperation(Guid[] ids, Power foreach (var id in ids) { - ManagedObjectReference vmReference = await GetVm(id); + var aggregate = await this.GetVm(id); + var vmReference = aggregate.MachineReference; if (vmReference == null) { @@ -292,10 +284,10 @@ public async Task BulkPowerOperation(Guid[] ids, Power switch (operation) { case PowerOperation.PowerOff: - taskReference = _client.PowerOffVM_TaskAsync(vmReference); + taskReference = aggregate.Connection.Client.PowerOffVM_TaskAsync(vmReference); break; case PowerOperation.PowerOn: - taskReference = _client.PowerOnVM_TaskAsync(vmReference, null); + taskReference = aggregate.Connection.Client.PowerOnVM_TaskAsync(vmReference, null); break; } @@ -319,7 +311,8 @@ public async Task RebootVm(Guid id) ManagedObjectReference vmReference = null; ManagedObjectReference task; - vmReference = await GetVm(id); + var aggregate = await this.GetVm(id); + vmReference = aggregate.MachineReference; if (vmReference == null) { @@ -334,9 +327,9 @@ public async Task RebootVm(Guid id) try { - task = await _client.PowerOffVM_TaskAsync(vmReference); + task = await aggregate.Connection.Client.PowerOffVM_TaskAsync(vmReference); - TaskInfo info = await WaitForVimTask(task); + TaskInfo info = await WaitForVimTask(task, aggregate.Connection); if (info.state != TaskInfoState.success) { return "error"; @@ -356,7 +349,8 @@ public async Task ShutdownVm(Guid id) ManagedObjectReference vmReference = null; string state = null; - vmReference = await GetVm(id); + var aggregate = await this.GetVm(id); + vmReference = aggregate.MachineReference; if (vmReference == null) { @@ -377,7 +371,7 @@ public async Task ShutdownVm(Guid id) try { - await _client.ShutdownGuestAsync(vmReference); + await aggregate.Connection.Client.ShutdownGuestAsync(vmReference); state = "shutdown submitted"; } catch (Exception ex) @@ -396,7 +390,8 @@ public async Task> BulkShutdown(Guid[] ids) foreach (var id in ids) { - ManagedObjectReference vmReference = await GetVm(id); + var aggregate = await this.GetVm(id); + var vmReference = aggregate.MachineReference; if (vmReference == null) { @@ -405,14 +400,14 @@ public async Task> BulkShutdown(Guid[] ids) continue; } - taskDict.Add(id, _client.ShutdownGuestAsync(vmReference)); + taskDict.Add(id, aggregate.Connection.Client.ShutdownGuestAsync(vmReference)); } try { await Task.WhenAll(taskDict.Values.Where(x => x != null)).ConfigureAwait(false); } - catch (Exception ex) + catch (Exception) { // Expected exception if shutdown failed, handled in finally } @@ -441,7 +436,8 @@ public async Task> BulkReboot(Guid[] ids) foreach (var id in ids) { - ManagedObjectReference vmReference = await GetVm(id); + var aggregate = await this.GetVm(id); + var vmReference = aggregate.MachineReference; if (vmReference == null) { @@ -450,14 +446,14 @@ public async Task> BulkReboot(Guid[] ids) continue; } - taskDict.Add(id, _client.RebootGuestAsync(vmReference)); + taskDict.Add(id, aggregate.Connection.Client.RebootGuestAsync(vmReference)); } try { await Task.WhenAll(taskDict.Values.Where(x => x != null)).ConfigureAwait(false); } - catch (Exception ex) + catch (Exception) { // Expected exception if reboot failed, handled in finally } @@ -537,11 +533,13 @@ public string GetPowerState(RetrievePropertiesResponse propertiesResponse) return state; } - public async Task GetPowerState(Guid uuid) + public async Task GetPowerState(Guid id) { _logger.LogDebug("GetPowerState called"); - ManagedObjectReference vmReference = await GetVm(uuid); + var aggregate = await this.GetVm(id); + var vmReference = aggregate.MachineReference; + if (vmReference == null) { _logger.LogDebug($"could not get state, vmReference is null"); @@ -549,8 +547,8 @@ public async Task GetPowerState(Guid uuid) } //retrieve the properties specified - RetrievePropertiesResponse response = await _client.RetrievePropertiesAsync( - _props, + RetrievePropertiesResponse response = await aggregate.Connection.Client.RetrievePropertiesAsync( + aggregate.Connection.Props, VmFilter(vmReference, "summary.runtime.powerState")); return GetPowerState(response); @@ -558,24 +556,40 @@ public async Task GetPowerState(Guid uuid) public async Task> GetPowerState(IEnumerable machineIds) { - var vmRefList = new List(); + var vmRefDict = new Dictionary>(); foreach (var id in machineIds) { - ManagedObjectReference vmReference = await GetVm(id); + var aggregate = await this.GetVm(id); + var vmReference = aggregate.MachineReference; if (vmReference != null) { - vmRefList.Add(vmReference); + if (!vmRefDict.ContainsKey(aggregate.Connection.Address)) + { + vmRefDict.Add(aggregate.Connection.Address, new List()); + } + + vmRefDict[aggregate.Connection.Address].Add(vmReference); } } - // retrieve the properties specified - RetrievePropertiesResponse response = await _client.RetrievePropertiesAsync( - _props, - VmFilter(vmRefList, "summary.runtime.powerState config.uuid")); + var connections = _connectionService.GetAllConnections(); + List> powerStates = new List>(); - return GetPowerStateMultiple(response); + foreach (var kvp in vmRefDict) + { + var connection = connections.Where(x => x.Address == kvp.Key).FirstOrDefault(); + + // retrieve the properties specified + RetrievePropertiesResponse response = await connection.Client.RetrievePropertiesAsync( + connection.Props, + VmFilter(kvp.Value, "summary.runtime.powerState config.uuid")); + + powerStates.Add(GetPowerStateMultiple(response)); + } + + return powerStates.SelectMany(x => x).ToDictionary(x => x.Key, y => y.Value); } private Dictionary GetPowerStateMultiple(RetrievePropertiesResponse propertiesResponse) @@ -637,10 +651,12 @@ public VirtualMachineToolsStatus GetVmToolsStatus(RetrievePropertiesResponse pro public async Task GetVmToolsStatus(Guid id) { - ManagedObjectReference vmReference = vmReference = await GetVm(id); + var aggregate = await GetVm(id); + var vmReference = aggregate.MachineReference; + //retrieve the properties specificied - RetrievePropertiesResponse response = await _client.RetrievePropertiesAsync( - _props, + RetrievePropertiesResponse response = await aggregate.Connection.Client.RetrievePropertiesAsync( + aggregate.Connection.Props, VmFilter(vmReference)); return GetVmToolsStatus(response); @@ -650,7 +666,8 @@ public async Task UploadFileToVm(Guid id, string username, string passwo { _logger.LogDebug("UploadFileToVm called"); - ManagedObjectReference vmReference = vmReference = await GetVm(id); + var aggregate = await GetVm(id); + var vmReference = aggregate.MachineReference; if (vmReference == null) { @@ -659,8 +676,8 @@ public async Task UploadFileToVm(Guid id, string username, string passwo return errorMessage; } //retrieve the properties specificied - RetrievePropertiesResponse response = await _client.RetrievePropertiesAsync( - _props, + RetrievePropertiesResponse response = await aggregate.Connection.Client.RetrievePropertiesAsync( + aggregate.Connection.Props, VmFilter(vmReference)); VimClient.ObjectContent[] oc = response.returnval; @@ -694,10 +711,10 @@ public async Task UploadFileToVm(Guid id, string username, string passwo }; // upload the file GuestFileAttributes fileAttributes = new GuestFileAttributes(); - var fileTransferUrl = _client.InitiateFileTransferToGuestAsync(fileManager, vmReference, credentialsAuth, filepath, fileAttributes, fileStream.Length, true).Result; + var fileTransferUrl = aggregate.Connection.Client.InitiateFileTransferToGuestAsync(fileManager, vmReference, credentialsAuth, filepath, fileAttributes, fileStream.Length, true).Result; // Replace IP address with hostname - RetrievePropertiesResponse hostResponse = await _client.RetrievePropertiesAsync(_props, HostFilter(vmSummary.runtime.host, "name")); + RetrievePropertiesResponse hostResponse = await aggregate.Connection.Client.RetrievePropertiesAsync(aggregate.Connection.Props, HostFilter(vmSummary.runtime.host, "name")); string hostName = hostResponse.returnval[0].propSet[0].val as string; if (!fileTransferUrl.Contains(hostName)) @@ -729,7 +746,7 @@ public async Task UploadFileToVm(Guid id, string username, string passwo return ""; } - private async Task WaitForVimTask(ManagedObjectReference task) + private async Task WaitForVimTask(ManagedObjectReference task, VsphereConnection connection) { int i = 0; TaskInfo info = new TaskInfo(); @@ -739,7 +756,7 @@ private async Task WaitForVimTask(ManagedObjectReference task) { //check every so often await Task.Delay(_pollInterval); - info = await GetVimTaskInfo(task); + info = await GetVimTaskInfo(task, connection); i++; //check for status updates until the task is complete } while ((info.state == TaskInfoState.running || info.state == TaskInfoState.queued)); @@ -748,11 +765,11 @@ private async Task WaitForVimTask(ManagedObjectReference task) return info; } - private async Task GetVimTaskInfo(ManagedObjectReference task) + private async Task GetVimTaskInfo(ManagedObjectReference task, VsphereConnection connection) { TaskInfo info = new TaskInfo(); - RetrievePropertiesResponse response = await _client.RetrievePropertiesAsync( - _props, + RetrievePropertiesResponse response = await connection.Client.RetrievePropertiesAsync( + connection.Props, TaskFilter(task)); VimClient.ObjectContent[] oc = response.returnval; info = (TaskInfo)oc[0].propSet[0].val; @@ -770,7 +787,8 @@ public async Task GetNicOptions(Guid id, bool canManage, IEnumerable public async Task> GetVmNetworks(VsphereVirtualMachine machine, bool canManage, IEnumerable allowedNetworks) { - List hostNetworks = _connectionService.GetNetworksByHost(machine.HostReference); + var aggregate = await this.GetVm(machine.Id); + List hostNetworks = _connectionService.GetNetworksByHost(machine.HostReference, aggregate.Connection.Address); List networkNames = hostNetworks.Select(n => n.Name).ToList(); // if a user can manage this VM, then they have access to all available NICs @@ -793,6 +811,7 @@ public async Task> GetVmNetworks(VsphereVirtualMachine machine, boo public async Task> GetVMConfiguration(VsphereVirtualMachine machine, Feature feature) { + var aggregate = await this.GetVm(machine.Id); VirtualDevice[] devices = machine.Devices; VirtualMachineConfigSpec vmcs = new VirtualMachineConfigSpec(); @@ -825,7 +844,7 @@ public async Task> GetVMConfiguration(VsphereVirtualM if (!string.IsNullOrEmpty(portGroupKey)) { - var network = _connectionService.GetNetworkByReference(portGroupKey); + var network = _connectionService.GetNetworkByReference(portGroupKey, aggregate.Connection.Address); string cardName = network?.Name; if (!string.IsNullOrEmpty(cardName)) @@ -850,26 +869,31 @@ public async Task> GetVMConfiguration(VsphereVirtualM return names; } - public async Task> GetIsos(string viewId, string subfolder) + public async Task> GetIsos(Guid vmId, string viewId, string subfolder) { + var aggregate = await this.GetVm(vmId); + var connection = aggregate.Connection; + List list = new List(); - var dsName = _options.DsName; - var baseFolder = _options.BaseFolder; + var dsName = connection.Host.DsName; + var baseFolder = connection.Host.BaseFolder; var filepath = $"[{dsName}] {baseFolder}/{viewId}/{subfolder}"; - var datastore = await GetDatastoreByName(dsName); + + var datastore = await GetDatastoreByName(dsName, connection); if (datastore == null) { - _logger.LogError($"Datastore {dsName} cannot be found."); + _logger.LogError($"Datastore {dsName} not found in {connection.Address}."); return list; } + var dsBrowser = datastore.Browser; ManagedObjectReference task = null; TaskInfo info = null; HostDatastoreBrowserSearchSpec spec = new HostDatastoreBrowserSearchSpec { }; List results = new List(); - task = await _client.SearchDatastore_TaskAsync(dsBrowser, filepath, spec); - info = await WaitForVimTask(task); + task = await connection.Client.SearchDatastore_TaskAsync(dsBrowser, filepath, spec); + info = await WaitForVimTask(task, connection); if (info.state == TaskInfoState.error) { if (info.error.fault != null && @@ -901,6 +925,7 @@ public async Task> GetIsos(string viewId, string subfolder) public async Task ReconfigureVm(Guid id, Feature feature, string label, string newvalue) { + var aggregate = await this.GetVm(id); VsphereVirtualMachine machine = await GetMachineById(id); ManagedObjectReference vmReference = machine.Reference; @@ -919,7 +944,7 @@ public async Task ReconfigureVm(Guid id, Feature feature, string label if (cdrom.backing.GetType() != typeof(VirtualCdromIsoBackingInfo)) cdrom.backing = new VirtualCdromIsoBackingInfo(); - ((VirtualCdromIsoBackingInfo)cdrom.backing).datastore = (await GetDatastoreByName(_options.DsName)).Reference; + ((VirtualCdromIsoBackingInfo)cdrom.backing).datastore = (await GetDatastoreByName(aggregate.Connection.Host.DsName, aggregate.Connection)).Reference; ((VirtualCdromIsoBackingInfo)cdrom.backing).fileName = newvalue; cdrom.connectable = new VirtualDeviceConnectInfo { @@ -945,7 +970,7 @@ public async Task ReconfigureVm(Guid id, Feature feature, string label if (card != null) { - Network network = _connectionService.GetNetworkByName(newvalue); + Network network = _connectionService.GetNetworkByName(newvalue, aggregate.Connection.Address); if (network.IsDistributed) { @@ -1007,8 +1032,8 @@ public async Task ReconfigureVm(Guid id, Feature feature, string label //break; } - ManagedObjectReference task = await _client.ReconfigVM_TaskAsync(vmReference, vmcs); - TaskInfo info = await WaitForVimTask(task); + ManagedObjectReference task = await aggregate.Connection.Client.ReconfigVM_TaskAsync(vmReference, vmcs); + TaskInfo info = await WaitForVimTask(task, aggregate.Connection); if (info.state == TaskInfoState.error) throw new Exception(info.error.localizedMessage); return info; @@ -1016,8 +1041,8 @@ public async Task ReconfigureVm(Guid id, Feature feature, string label public async Task SetResolution(Guid id, int width, int height) { - - ManagedObjectReference vmReference = await GetVm(id); + var aggregate = await this.GetVm(id); + var vmReference = aggregate.MachineReference; string state = await GetPowerState(id); if (vmReference == null) @@ -1037,7 +1062,7 @@ public async Task SetResolution(Guid id, int width, int height) try { - await _client.SetScreenResolutionAsync(vmReference, width, height); + await aggregate.Connection.Client.SetScreenResolutionAsync(vmReference, width, height); state = "set resolution submitted"; } catch (Exception ex) @@ -1049,69 +1074,29 @@ public async Task SetResolution(Guid id, int width, int height) return state; } - public async Task> GetEvents(EventFilterSpec filterSpec) + public async Task> GetEvents(EventFilterSpec filterSpec, VsphereConnection connection) { var events = new List(); const int maxCount = 1000; // maximum allowable by vsphere api - if (_client != null) + if (connection.Client != null) { - var collector = await _client.CreateCollectorForEventsAsync(_sic.eventManager, filterSpec); + var collector = await connection.Client.CreateCollectorForEventsAsync(connection.Sic.eventManager, filterSpec); int resultCount; do { - var response = await _client.ReadNextEventsAsync(collector, maxCount); + var response = await connection.Client.ReadNextEventsAsync(collector, maxCount); events.AddRange(response.returnval); resultCount = response.returnval.Length; } while (resultCount != 0); - await _client.DestroyCollectorAsync(collector); + await connection.Client.DestroyCollectorAsync(collector); } return events; } - #region Helpers - private bool IsCardDistributed(string cardName) - { - if (_networks != null && !string.IsNullOrEmpty(cardName)) - { - NetworkMOR card; - if ((card = _networks.Where(n => n.Name.Equals(cardName)).SingleOrDefault()) != null) - { - return card.Type.Equals("DistributedVirtualPortgroup"); - } - } - return false; - } - private string GetPortGroupId(string cardName) - { - if (_networks != null && !string.IsNullOrEmpty(cardName)) - { - NetworkMOR card; - if ((card = _networks.Where(n => n.Name.Equals(cardName)).SingleOrDefault()) != null) - { - return card.Value; - } - } - return string.Empty; - } - private string GetSwitchUuid(string cardName) - { - if (_networks != null && !string.IsNullOrEmpty(cardName)) - { - NetworkMOR card; - if ((card = _networks.Where(n => n.Name.Equals(cardName)).SingleOrDefault()) != null) - { - return card.Uuid; - } - } - return string.Empty; - } - - #endregion - #region Filters public static PropertyFilterSpec[] TaskFilter(ManagedObjectReference mor) @@ -1135,96 +1120,6 @@ public static PropertyFilterSpec[] TaskFilter(ManagedObjectReference mor) }; } - public PropertyFilterSpec[] NetworkSearchFilter() - { - PropertySpec prop; - List props = new List(); - - TraversalSpec trav = new TraversalSpec(); - List list = new List(); - - SelectionSpec sel = new SelectionSpec(); - List selectset = new List(); - - ObjectSpec objectspec = new ObjectSpec(); - PropertyFilterSpec filter = new PropertyFilterSpec(); - - trav.name = "DatacenterTraversalSpec"; - trav.type = "Datacenter"; - trav.path = "networkFolder"; - - sel.name = "FolderTraversalSpec"; - selectset.Add(sel); - trav.selectSet = selectset.ToArray(); - list.Add(trav); - - trav = new TraversalSpec(); - trav.name = "FolderTraversalSpec"; - trav.type = "Folder"; - trav.path = "childEntity"; - selectset.Clear(); - sel = new SelectionSpec(); - sel.name = "DatacenterTraversalSpec"; - selectset.Add(sel); - trav.selectSet = selectset.ToArray(); - list.Add(trav); - - prop = new PropertySpec(); - prop.type = "Datacenter"; - prop.pathSet = new string[] { "networkFolder", "name" }; - props.Add(prop); - - prop = new PropertySpec(); - prop.type = "Folder"; - prop.pathSet = new string[] { "childEntity", "name" }; - props.Add(prop); - - prop = new PropertySpec(); - prop.type = "VmwareDistributedVirtualSwitch"; - prop.pathSet = new string[] { "portgroup", "name", "parent", "uuid" }; - props.Add(prop); - - prop = new PropertySpec(); - prop.type = "DistributedVirtualPortgroup"; - prop.pathSet = new string[] { "name", "key" }; - props.Add(prop); - - objectspec = new ObjectSpec(); - objectspec.obj = _sic.rootFolder; - objectspec.selectSet = list.ToArray(); - - filter = new PropertyFilterSpec(); - filter.propSet = props.ToArray(); - filter.objectSet = new ObjectSpec[] { objectspec }; - PropertyFilterSpec[] _dvNetworkSearchFilters = new PropertyFilterSpec[] { filter }; - return _dvNetworkSearchFilters; - } - - public static PropertyFilterSpec[] NetworkFilter(ManagedObjectReference mor) - { - return NetworkFilter(mor, "networkInfo.dnsConfig networkInfo.ipRouteConfig networkInfo.portgroup networkInfo.vnic networkInfo.vswitch"); - } - public static PropertyFilterSpec[] NetworkFilter(ManagedObjectReference mor, string props) - { - PropertySpec prop = new PropertySpec - { - type = "HostNetworkSystem", - pathSet = props.Split(new char[] { ' ', ',' }, StringSplitOptions.RemoveEmptyEntries) - }; - - ObjectSpec objectspec = new ObjectSpec - { - obj = mor, //_net - }; - - return new PropertyFilterSpec[] { - new PropertyFilterSpec { - propSet = new PropertySpec[] { prop }, - objectSet = new ObjectSpec[] { objectspec } - } - }; - } - public static PropertyFilterSpec[] VmFilter(ManagedObjectReference mor) { return VmFilter(new ManagedObjectReference[] { mor }, "summary"); @@ -1299,20 +1194,6 @@ public static PropertyFilterSpec[] HostFilter(ManagedObjectReference mor, string return ret; } - private async Task GetDistributedSwitchUuid(ManagedObjectReference mor) - { - RetrievePropertiesResponse response = await _client.RetrievePropertiesAsync(_props, PortGroupFilter(mor)); - ManagedObjectReference switchMOR = ((DVPortgroupConfigInfo)response.returnval[0].propSet[0].val).distributedVirtualSwitch; - response = await _client.RetrievePropertiesAsync(_props, SwitchFilter(switchMOR)); - - var config = ((VMwareDVSConfigInfo)response.returnval[0].GetProperty("config")); - - // if this is an uplink switch, return null so that we don't use it as a NIC - if ((config.uplinkPortgroup[0].Value.Equals(mor.Value))) - return null; - return ((string)response.returnval[0].GetProperty("uuid")); - } - public static PropertyFilterSpec[] SwitchFilter(ManagedObjectReference mor) { PropertySpec prop = new PropertySpec @@ -1416,28 +1297,12 @@ public static PropertyFilterSpec[] DatastoreFilter(ManagedObjectReference mor) public async Task GetMachineById(Guid id) { - ManagedObjectReference machineReference = _connectionService.GetMachineById(id); - - if (machineReference == null) - { - // lookup reference - machineReference = await GetVm(id); - - // return null if not found - if (machineReference == null) - { - return null; - } - } - - if (_client == null) - { - return null; - } + var aggregate = await this.GetVm(id); + var machineReference = aggregate.MachineReference; // retrieve all machine properties we need - RetrievePropertiesResponse propertiesResponse = await _client.RetrievePropertiesAsync( - _props, + RetrievePropertiesResponse propertiesResponse = await aggregate.Connection.Client.RetrievePropertiesAsync( + aggregate.Connection.Props, VmFilter(machineReference, "name summary.guest.toolsStatus summary.runtime.host summary.runtime.powerState config.hardware.device")); VimClient.ObjectContent vm = propertiesResponse.returnval.FirstOrDefault(); @@ -1463,14 +1328,22 @@ public async Task GetMachineById(Guid id) return machine; } - private async Task GetDatastoreByName(string dsName) + private async Task GetDatastoreByName(string dsName, VsphereConnection connection) { - Datastore datastore = _connectionService.GetDatastoreByName(dsName); + Datastore datastore = _connectionService.GetDatastoreByName(dsName, connection.Address); if (datastore == null) { - // lookup reference - datastore = await GetNewDatastore(dsName); + try + { + // lookup reference + datastore = await GetNewDatastore(dsName, connection); + } + catch (Exception ex) + { + datastore = null; + _logger.LogError(ex, $"Datastore {dsName} not found in {connection.Address}"); + } // return null if not found if (datastore == null) @@ -1479,7 +1352,7 @@ private async Task GetDatastoreByName(string dsName) } } - if (_client == null) + if (connection.Client == null) { return null; } @@ -1487,9 +1360,9 @@ private async Task GetDatastoreByName(string dsName) return datastore; } - private async Task GetNewDatastore(string dsName) + private async Task GetNewDatastore(string dsName, VsphereConnection connection) { - var clunkyTree = await LoadReferenceTree(_client); + var clunkyTree = await LoadReferenceTree(connection); if (clunkyTree.Length == 0) { throw new InvalidOperationException(); @@ -1510,7 +1383,7 @@ private async Task GetNewDatastore(string dsName) return null; } - private async Task LoadReferenceTree(VimPortTypeClient client) + private async Task LoadReferenceTree(VsphereConnection connection) { var plan = new TraversalSpec { @@ -1553,7 +1426,7 @@ private async Task GetNewDatastore(string dsName) }; ObjectSpec objectspec = new ObjectSpec(); - objectspec.obj = _sic.rootFolder; + objectspec.obj = connection.Sic.rootFolder; objectspec.selectSet = new SelectionSpec[] { plan }; PropertyFilterSpec filter = new PropertyFilterSpec(); @@ -1561,7 +1434,7 @@ private async Task GetNewDatastore(string dsName) filter.objectSet = new ObjectSpec[] { objectspec }; PropertyFilterSpec[] filters = new PropertyFilterSpec[] { filter }; - RetrievePropertiesResponse response = await client.RetrievePropertiesAsync(_props, filters); + RetrievePropertiesResponse response = await connection.Client.RetrievePropertiesAsync(connection.Props, filters); return response.returnval; } diff --git a/src/Player.Vm.Api/Features/Vsphere/Queries/Get.cs b/src/Player.Vm.Api/Features/Vsphere/Queries/Get.cs index e21ab66..bf53900 100644 --- a/src/Player.Vm.Api/Features/Vsphere/Queries/Get.cs +++ b/src/Player.Vm.Api/Features/Vsphere/Queries/Get.cs @@ -18,6 +18,9 @@ using Player.Vm.Api.Infrastructure.Extensions; using Player.Vm.Api.Domain.Services; using Player.Api.Client; +using System.Collections.Generic; +using Microsoft.Extensions.Options; +using System.Linq; namespace Player.Vm.Api.Features.Vsphere { @@ -38,8 +41,6 @@ public class Handler : BaseHandler, IRequestHandler _logger; private readonly VsphereOptions _vsphereOptions; private readonly ClaimsPrincipal _user; - private readonly IPlayerApiClient _playerClient; - private readonly IPlayerService _playerService; public Handler( IVmService vmService, @@ -48,7 +49,6 @@ public Handler( ILogger logger, VsphereOptions vsphereOptions, IPrincipal user, - IPlayerApiClient playerClient, IPlayerService playerService, IPermissionsService permissionsService) : base(mapper, vsphereService, playerService, user, permissionsService, vmService) @@ -59,7 +59,6 @@ public Handler( _logger = logger; _vsphereOptions = vsphereOptions; _user = user as ClaimsPrincipal; - _playerClient = playerClient; } public async Task Handle(Query request, CancellationToken cancellationToken) diff --git a/src/Player.Vm.Api/Features/Vsphere/Queries/GetIsos.cs b/src/Player.Vm.Api/Features/Vsphere/Queries/GetIsos.cs index 0debfc4..a8e34d5 100644 --- a/src/Player.Vm.Api/Features/Vsphere/Queries/GetIsos.cs +++ b/src/Player.Vm.Api/Features/Vsphere/Queries/GetIsos.cs @@ -75,7 +75,7 @@ public async Task Handle(Query request, CancellationToken cancellat foreach (var viewId in viewIds) { - isoTasks.Add(this.GetViewIsos(viewId, cancellationToken)); + isoTasks.Add(this.GetViewIsos(vm.Id, viewId, cancellationToken)); } await Task.WhenAll(isoTasks); @@ -91,14 +91,14 @@ public async Task Handle(Query request, CancellationToken cancellat return results.ToArray(); } - private async Task GetViewIsos(Guid viewId, CancellationToken cancellationToken) + private async Task GetViewIsos(Guid vmId, Guid viewId, CancellationToken cancellationToken) { var teams = await _playerService.GetTeamsByViewIdAsync(viewId, cancellationToken); // User has access to this view if (teams.Count() > 0) { - return await this.GetViewIsos(viewId, teams, cancellationToken); + return await this.GetViewIsos(vmId, viewId, teams, cancellationToken); } else { @@ -106,16 +106,16 @@ private async Task GetViewIsos(Guid viewId, CancellationToken cancell } } - private async Task GetViewIsos(Guid viewId, IEnumerable teams, CancellationToken cancellationToken) + private async Task GetViewIsos(Guid vmId, Guid viewId, IEnumerable teams, CancellationToken cancellationToken) { var viewTask = _playerService.GetViewByIdAsync(viewId, cancellationToken); var isoTaskDict = new Dictionary>>(); - isoTaskDict.Add(viewId, _vsphereService.GetIsos(viewId.ToString(), viewId.ToString())); + isoTaskDict.Add(viewId, _vsphereService.GetIsos(vmId, viewId.ToString(), viewId.ToString())); foreach (var team in teams) { - isoTaskDict.Add(team.Id, _vsphereService.GetIsos(viewId.ToString(), team.Id.ToString())); + isoTaskDict.Add(team.Id, _vsphereService.GetIsos(vmId, viewId.ToString(), team.Id.ToString())); } var tasks = new List(); diff --git a/src/Player.Vm.Api/Infrastructure/Extensions/AsyncExExtensions.cs b/src/Player.Vm.Api/Infrastructure/Extensions/AsyncExExtensions.cs index 353b402..c03469a 100644 --- a/src/Player.Vm.Api/Infrastructure/Extensions/AsyncExExtensions.cs +++ b/src/Player.Vm.Api/Infrastructure/Extensions/AsyncExExtensions.cs @@ -22,7 +22,7 @@ public static async Task WaitAsync(this AsyncAutoResetEvent mEvent, TimeSp await mEvent.WaitAsync(comp.Token).ConfigureAwait(false); return true; } - catch (OperationCanceledException e) + catch (OperationCanceledException) { if (token.IsCancellationRequested) throw; //Forward OperationCanceledException from external Token diff --git a/src/Player.Vm.Api/Properties/launchSettings.json b/src/Player.Vm.Api/Properties/launchSettings.json index af7dcc5..58fbf1a 100644 --- a/src/Player.Vm.Api/Properties/launchSettings.json +++ b/src/Player.Vm.Api/Properties/launchSettings.json @@ -27,4 +27,4 @@ "applicationUrl": "http://localhost:4302/" } } -} \ No newline at end of file +} diff --git a/src/Player.Vm.Api/appsettings.json b/src/Player.Vm.Api/appsettings.json index 650d0a1..d8cc2e7 100644 --- a/src/Player.Vm.Api/appsettings.json +++ b/src/Player.Vm.Api/appsettings.json @@ -57,20 +57,23 @@ "MaxFileSize": 6000000000 }, "Vsphere": { - "Enabled": true, - "Host": "", - "Username": "", - "Password": "", - "DsName": "", - "BaseFolder": "player", - "Timeout": 30, "ConnectionRetryIntervalSeconds": 60, "ConnectionRefreshIntervalMinutes": 20, "LoadCacheAfterIterations": 5, "LogConsoleAccess": false, "CheckTaskProgressIntervalMilliseconds": 5000, "ReCheckTaskProgressIntervalMilliseconds": 1000, - "HealthAllowanceSeconds": 180 + "HealthAllowanceSeconds": 180, + "Hosts": [ + { + "Enabled": true, + "Address": "", + "Username": "", + "Password": "", + "DsName": "", + "BaseFolder": "player" + } + ] }, "RewriteHost": { "RewriteHost": false,