From d77ab7283da0045c276735b5b95f38303c38f04c Mon Sep 17 00:00:00 2001 From: Laila Kassar Date: Thu, 5 Aug 2021 16:22:25 +0000 Subject: [PATCH 1/3] Created preliminary useragent cloud creation --- pkg/azuredisk/azuredisk.go | 1 + pkg/azuredisk/controllerserver.go | 89 +++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/pkg/azuredisk/azuredisk.go b/pkg/azuredisk/azuredisk.go index 216f309bce..bff6bb3c82 100644 --- a/pkg/azuredisk/azuredisk.go +++ b/pkg/azuredisk/azuredisk.go @@ -98,6 +98,7 @@ const ( networkAccessPolicyField = "networkaccesspolicy" diskAccessIDField = "diskaccessid" enableBurstingField = "enablebursting" + userAgentField = "useragent" WellKnownTopologyKey = "topology.kubernetes.io/zone" throttlingKey = "throttlingKey" diff --git a/pkg/azuredisk/controllerserver.go b/pkg/azuredisk/controllerserver.go index 73224b575a..64fb7ffe3a 100644 --- a/pkg/azuredisk/controllerserver.go +++ b/pkg/azuredisk/controllerserver.go @@ -17,8 +17,13 @@ limitations under the License. package azuredisk import ( + "bytes" "context" + "encoding/json" "fmt" + "io" + "io/ioutil" + "os" "path" "sort" "strconv" @@ -40,6 +45,7 @@ import ( "k8s.io/klog/v2" "sigs.k8s.io/azuredisk-csi-driver/pkg/optimization" + "sigs.k8s.io/azuredisk-csi-driver/pkg/util" volumehelper "sigs.k8s.io/azuredisk-csi-driver/pkg/util" "sigs.k8s.io/cloud-provider-azure/pkg/metrics" azure "sigs.k8s.io/cloud-provider-azure/pkg/provider" @@ -293,6 +299,14 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) contentSource := &csi.VolumeContentSource{} for k, v := range customTagsMap { tags[k] = v + if k == userAgentField { + temp := d.cloud + d.cloud, err = SetupNewUserAgentCloud(v, d.cloud) + if err != nil { + klog.V(2).Infof("Unable to create new cloud for UserAgent, err: (%s)", err) + } + d.cloud = temp + } } if strings.EqualFold(writeAcceleratorEnabled, trueValue) { @@ -1219,3 +1233,78 @@ func getEntriesAndNextToken(req *csi.ListSnapshotsRequest, snapshots []compute.S return listSnapshotResp, nil } + +/// Creates a new cloud to configure the Useragent property of DiskClient +func SetupNewUserAgentCloud(userAgent string, currentAz *azure.Cloud) (*azure.Cloud, error) { + //TODO + az := &azure.Cloud{ + InitSecretConfig: currentAz.InitSecretConfig, + } + + //TODO how to check the kubeconfig to see if it is from secret or not? + + credFile, ok := os.LookupEnv(DefaultAzureCredentialFileEnv) + if ok && strings.TrimSpace(credFile) != "" { + klog.V(2).Infof("%s env var set as %v", DefaultAzureCredentialFileEnv, credFile) + } else { + if util.IsWindowsOS() { + credFile = DefaultCredFilePathWindows + } else { + credFile = DefaultCredFilePathLinux + } + + klog.V(2).Infof("use default %s env var: %v", DefaultAzureCredentialFileEnv, credFile) + } + + var config io.Reader + + config, err := UpdateConfigUserAgent(userAgent, credFile) + if err != nil { + klog.Errorf("updating azure config file(%s) with useragent(%s) failed with %v", credFile, userAgent, err) + return nil, fmt.Errorf("updating azure config file(%s) with useragent(%s) failed with %v", credFile, userAgent, err) + } + + klog.V(2).Infof("read cloud config from file: %s and set useragent: %s successfully", credFile, userAgent) + if az, err = azure.NewCloudWithoutFeatureGates(config, false); err != nil { + return az, err + } + + return az, nil +} + +func UpdateConfigUserAgent(userAgent string, credFile string) (io.Reader, error) { + var configReader *os.File + configReader, err := os.Open(credFile) + if err != nil { + klog.Errorf("load azure config from file(%s) failed with %v", credFile, err) + return nil, err + } + var config azure.Config + if configReader == nil { + return nil, nil + } + + configContents, err := ioutil.ReadAll(configReader) + if err != nil { + return nil, err + } + + err = json.Unmarshal(configContents, &config) + if err != nil { + return nil, err + } + + config.UserAgent = userAgent + newConfig, err := json.Marshal(config) + if err != nil { + return nil, err + } + + var newReader io.Reader + buf := bytes.NewBuffer(newConfig) + newReader = buf + + configReader.Close() + + return newReader, nil +} From d0fc2267a3a386462de3e0f7cb43ad7cfb1943e2 Mon Sep 17 00:00:00 2001 From: Laila Kassar Date: Tue, 10 Aug 2021 12:25:24 +0000 Subject: [PATCH 2/3] Update cloud-provider-azure vendor --- .../pkg/provider/azure.go | 40 +++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go index 644082e8a9..820c47582c 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure.go @@ -41,6 +41,7 @@ import ( v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/flowcontrol" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog/v2" @@ -225,6 +226,8 @@ type Config struct { // RouteUpdateWaitingInSeconds is the delay time for waiting route updates to take effect. This waiting delay is added // because the routes are not taken effect when the async route updating operation returns success. Default is 30 seconds. RouteUpdateWaitingInSeconds int `json:"routeUpdateWaitingInSeconds,omitempty" yaml:"routeUpdateWaitingInSeconds,omitempty"` + // The user agent for Azure customer usage attribution + UserAgent string `json:"userAgent,omitempty" yaml:"userAgent,omitempty"` } type InitSecretConfig struct { @@ -573,14 +576,18 @@ func (az *Cloud) InitializeCloudFromConfig(config *Config, fromSecret, callFromC az.routeUpdater = newDelayedRouteUpdater(az, routeUpdateInterval) go az.routeUpdater.run() - // wait for the success first time of syncing zones - err = az.syncRegionZonesMap() - if err != nil { - klog.Errorf("InitializeCloudFromConfig: failed to sync regional zones map for the first time: %s", err.Error()) - return err - } + // Azure Stack does not support zone at the moment + // https://docs.microsoft.com/en-us/azure-stack/user/azure-stack-network-differences?view=azs-2102 + if !az.isStackCloud() { + // wait for the success first time of syncing zones + err = az.syncRegionZonesMap() + if err != nil { + klog.Errorf("InitializeCloudFromConfig: failed to sync regional zones map for the first time: %s", err.Error()) + return err + } - go az.refreshZones(az.syncRegionZonesMap) + go az.refreshZones(az.syncRegionZonesMap) + } } return nil @@ -780,6 +787,7 @@ func (az *Cloud) getAzureClientConfig(servicePrincipalToken *adal.ServicePrincip Authorizer: autorest.NewBearerAuthorizer(servicePrincipalToken), Backoff: &retry.Backoff{Steps: 1}, DisableAzureStackCloud: az.Config.DisableAzureStackCloud, + UserAgent: az.Config.UserAgent, } if az.Config.CloudProviderBackoff { @@ -824,6 +832,10 @@ func parseConfig(configReader io.Reader) (*Config, error) { return &config, nil } +func (az *Cloud) isStackCloud() bool { + return strings.EqualFold(az.Config.Cloud, consts.AzureStackCloudName) && !az.Config.DisableAzureStackCloud +} + // Initialize passes a Kubernetes clientBuilder interface to the cloud provider func (az *Cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, stop <-chan struct{}) { az.KubeClient = clientBuilder.ClientOrDie("azure-cloud-provider") @@ -849,6 +861,11 @@ func (az *Cloud) InstancesV2() (cloudprovider.InstancesV2, bool) { // Zones returns a zones interface. Also returns true if the interface is supported, false otherwise. func (az *Cloud) Zones() (cloudprovider.Zones, bool) { + if az.isStackCloud() { + // Azure stack does not support zones at this point + // https://docs.microsoft.com/en-us/azure-stack/user/azure-stack-network-differences?view=azs-2102 + return nil, false + } return az, true } @@ -876,6 +893,14 @@ func initDiskControllers(az *Cloud) error { // Common controller contains the function // needed by both blob disk and managed disk controllers + qps := float32(defaultAtachDetachDiskQPS) + bucket := defaultAtachDetachDiskBucket + if az.Config.AttachDetachDiskRateLimit != nil { + qps = az.Config.AttachDetachDiskRateLimit.CloudProviderRateLimitQPSWrite + bucket = az.Config.AttachDetachDiskRateLimit.CloudProviderRateLimitBucketWrite + } + klog.V(2).Infof("attach/detach disk operation rate limit QPS: %f, Bucket: %d", qps, bucket) + common := &controllerCommon{ location: az.Location, storageEndpointSuffix: az.Environment.StorageEndpointSuffix, @@ -883,6 +908,7 @@ func initDiskControllers(az *Cloud) error { subscriptionID: az.SubscriptionID, cloud: az, lockMap: newLockMap(), + diskOpRateLimiter: flowcontrol.NewTokenBucketRateLimiter(qps, bucket), } if az.HasExtendedLocation() { From 092b41dc3a87b55655fccfe048c1fd0e8ce1e074 Mon Sep 17 00:00:00 2001 From: Laila Kassar Date: Mon, 16 Aug 2021 22:20:44 +0000 Subject: [PATCH 3/3] Updated vendor pkg for cloud-provider-azure/pkg/provider, updated controllerserver.go to use storageclass variable and include config from secret --- pkg/azuredisk/controllerserver.go | 135 +++++++++++++----- .../pkg/provider/azure_backoff.go | 4 +- .../pkg/provider/azure_controller_common.go | 70 +++++++-- .../pkg/provider/azure_controller_standard.go | 34 +++-- .../pkg/provider/azure_controller_vmss.go | 34 +++-- .../pkg/provider/azure_fakes.go | 2 + .../pkg/provider/azure_instances.go | 37 +++-- .../pkg/provider/azure_loadbalancer.go | 65 ++++----- .../pkg/provider/azure_ratelimit.go | 13 ++ .../pkg/provider/azure_standard.go | 16 ++- .../pkg/provider/azure_storageaccount.go | 91 ++++++------ .../pkg/provider/azure_vmsets.go | 19 ++- .../pkg/provider/azure_vmss.go | 24 ++++ .../pkg/provider/azure_zones.go | 7 + .../cloud-provider-azure/pkg/provider/doc.go | 2 +- 15 files changed, 387 insertions(+), 166 deletions(-) diff --git a/pkg/azuredisk/controllerserver.go b/pkg/azuredisk/controllerserver.go index 64fb7ffe3a..57e829a85a 100644 --- a/pkg/azuredisk/controllerserver.go +++ b/pkg/azuredisk/controllerserver.go @@ -29,6 +29,8 @@ import ( "strconv" "strings" + "sigs.k8s.io/yaml" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-12-01/compute" "github.com/Azure/go-autorest/autorest/to" "github.com/container-storage-interface/spec/lib/go/csi" @@ -140,6 +142,7 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) diskAccessID string maxShares int enableBursting *bool + nonUserAgentCloud *azure.Cloud ) tags := make(map[string]string) @@ -207,6 +210,15 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) if strings.EqualFold(v, trueValue) { enableBursting = to.BoolPtr(true) } + case userAgentField: + nonUserAgentCloud = d.cloud + newUserAgent := v + d.cloud, err = SetupNewUserAgentCloud(newUserAgent, d.cloud) + if err != nil { + klog.V(2).Infof("Unable to create new cloud for UserAgent, err: (%s)", err) + d.cloud = nonUserAgentCloud + nonUserAgentCloud = nil + } default: return nil, fmt.Errorf("invalid parameter %s in storage class", k) } @@ -299,14 +311,6 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) contentSource := &csi.VolumeContentSource{} for k, v := range customTagsMap { tags[k] = v - if k == userAgentField { - temp := d.cloud - d.cloud, err = SetupNewUserAgentCloud(v, d.cloud) - if err != nil { - klog.V(2).Infof("Unable to create new cloud for UserAgent, err: (%s)", err) - } - d.cloud = temp - } } if strings.EqualFold(writeAcceleratorEnabled, trueValue) { @@ -376,6 +380,10 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) return nil, err } + if nonUserAgentCloud != nil { + d.cloud = nonUserAgentCloud + } + isOperationSucceeded = true klog.V(2).Infof("create azure disk(%s) account type(%s) rg(%s) location(%s) size(%d) tags(%s) successfully", diskName, skuName, resourceGroup, location, requestGiB, tags) @@ -1234,44 +1242,53 @@ func getEntriesAndNextToken(req *csi.ListSnapshotsRequest, snapshots []compute.S return listSnapshotResp, nil } -/// Creates a new cloud to configure the Useragent property of DiskClient +// Creates a new cloud during CreateVolume call to configure the Useragent property of DiskClient func SetupNewUserAgentCloud(userAgent string, currentAz *azure.Cloud) (*azure.Cloud, error) { - //TODO + kubeClient := currentAz.KubeClient az := &azure.Cloud{ InitSecretConfig: currentAz.InitSecretConfig, } - - //TODO how to check the kubeconfig to see if it is from secret or not? - - credFile, ok := os.LookupEnv(DefaultAzureCredentialFileEnv) - if ok && strings.TrimSpace(credFile) != "" { - klog.V(2).Infof("%s env var set as %v", DefaultAzureCredentialFileEnv, credFile) - } else { - if util.IsWindowsOS() { - credFile = DefaultCredFilePathWindows - } else { - credFile = DefaultCredFilePathLinux + if kubeClient != nil { + klog.V(2).Infof("reading cloud config from secret") + az.KubeClient = kubeClient + if err := InitializeCloudFromSecretWithUserAgent(az, userAgent); err != nil { + klog.V(2).Infof("InitializeCloudFromSecret with useragent %s failed with error: %v", userAgent, err) } - - klog.V(2).Infof("use default %s env var: %v", DefaultAzureCredentialFileEnv, credFile) } + if az.TenantID == "" || az.SubscriptionID == "" || az.ResourceGroup == "" { + klog.V(2).Infof("could not read cloud config from secret") + credFile, ok := os.LookupEnv(DefaultAzureCredentialFileEnv) + if ok && strings.TrimSpace(credFile) != "" { + klog.V(2).Infof("%s env var set as %v", DefaultAzureCredentialFileEnv, credFile) + } else { + if util.IsWindowsOS() { + credFile = DefaultCredFilePathWindows + } else { + credFile = DefaultCredFilePathLinux + } + klog.V(2).Infof("use default %s env var: %v", DefaultAzureCredentialFileEnv, credFile) + } + var config io.Reader - var config io.Reader + config, err := UpdateConfigUserAgent(userAgent, credFile) + if err != nil { + klog.Errorf("creating azure config from file(%s) with useragent(%s) failed with %v", credFile, userAgent, err) + return nil, fmt.Errorf("creating azure config from file(%s) with useragent(%s) failed with %v", credFile, userAgent, err) + } - config, err := UpdateConfigUserAgent(userAgent, credFile) - if err != nil { - klog.Errorf("updating azure config file(%s) with useragent(%s) failed with %v", credFile, userAgent, err) - return nil, fmt.Errorf("updating azure config file(%s) with useragent(%s) failed with %v", credFile, userAgent, err) + klog.V(2).Infof("created azure cloud config from file: %s and set useragent: %s successfully", credFile, userAgent) + if az, err = azure.NewCloudWithoutFeatureGates(config, false); err != nil { + return az, err + } } - - klog.V(2).Infof("read cloud config from file: %s and set useragent: %s successfully", credFile, userAgent) - if az, err = azure.NewCloudWithoutFeatureGates(config, false); err != nil { - return az, err + // reassign kubeClient + if kubeClient != nil && az.KubeClient == nil { + az.KubeClient = kubeClient } - return az, nil } +// Creates a config buffer that includes a useragent to be used for cloud creation func UpdateConfigUserAgent(userAgent string, credFile string) (io.Reader, error) { var configReader *os.File configReader, err := os.Open(credFile) @@ -1308,3 +1325,55 @@ func UpdateConfigUserAgent(userAgent string, credFile string) (io.Reader, error) return newReader, nil } + +// InitializeCloudFromSecret initializes Azure cloud provider from Kubernetes secret. +func InitializeCloudFromSecretWithUserAgent(az *azure.Cloud, userAgent string) error { + config, err := getConfigFromSecretWithUserAgent(az, userAgent) + if err != nil { + klog.Errorf("Failed to get cloud-config from secret: %v", err) + return fmt.Errorf("InitializeCloudFromSecret: failed to get cloud config from secret %s/%s: %w", az.SecretNamespace, az.SecretName, err) + } + + if config == nil { + // Skip re-initialization if the config is not override. + return nil + } + + if err := az.InitializeCloudFromConfig(config, true, true); err != nil { + klog.Errorf("Failed to initialize Azure cloud provider: %v with useragent", err, userAgent) + return fmt.Errorf("InitializeCloudFromSecret: failed to initialize Azure cloud provider: %w with useragent: %s", err, userAgent) + } + + return nil +} + +func getConfigFromSecretWithUserAgent(az *azure.Cloud, userAgent string) (*azure.Config, error) { + // Read config from file and no override, return nil. + if az.Config.CloudConfigType == "file" { + return nil, nil + } + + secret, err := az.KubeClient.CoreV1().Secrets(az.SecretNamespace).Get(context.TODO(), az.SecretName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get secret %s/%s: %w", az.SecretNamespace, az.SecretName, err) + } + + cloudConfigData, ok := secret.Data[az.CloudConfigKey] + if !ok { + return nil, fmt.Errorf("cloud-config is not set in the secret (%s/%s)", az.SecretNamespace, az.SecretName) + } + + config := azure.Config{} + if az.Config.CloudConfigType == "" || az.Config.CloudConfigType == "merge" { + // Merge cloud config, set default value to existing config. + config = az.Config + } + + err = yaml.Unmarshal(cloudConfigData, &config) + if err != nil { + return nil, fmt.Errorf("failed to parse Azure cloud-config: %w", err) + } + config.UserAgent = userAgent + + return &config, nil +} diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_backoff.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_backoff.go index fd01398b2d..9e8dea4eb9 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_backoff.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_backoff.go @@ -384,7 +384,7 @@ func (az *Cloud) DeletePublicIP(service *v1.Service, pipResourceGroup string, pi } // DeleteLB invokes az.LoadBalancerClient.Delete with exponential backoff retry -func (az *Cloud) DeleteLB(service *v1.Service, lbName string) error { +func (az *Cloud) DeleteLB(service *v1.Service, lbName string) *retry.Error { ctx, cancel := getContextWithCancel() defer cancel() @@ -398,7 +398,7 @@ func (az *Cloud) DeleteLB(service *v1.Service, lbName string) error { klog.Errorf("LoadBalancerClient.Delete(%s) failed: %s", lbName, rerr.Error().Error()) az.Event(service, v1.EventTypeWarning, "DeleteLoadBalancer", rerr.Error().Error()) - return rerr.Error() + return rerr } // CreateOrUpdateRouteTable invokes az.RouteTablesClient.CreateOrUpdate with exponential backoff retry diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_common.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_common.go index 154dbed1db..d96f203ee8 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_common.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_common.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/types" kwait "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/flowcontrol" cloudprovider "k8s.io/cloud-provider" volerr "k8s.io/cloud-provider/volume/errors" "k8s.io/klog/v2" @@ -95,6 +96,8 @@ type controllerCommon struct { // > attachDiskMap sync.Map detachDiskMap sync.Map + // attach/detach disk rate limiter + diskOpRateLimiter flowcontrol.RateLimiter } // AttachDiskOptions attach disk options @@ -162,13 +165,13 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri return -1, err } if strings.EqualFold(string(nodeName), string(attachedNode)) { - err := fmt.Errorf("volume %q is actually attached to current node %q, invalidate vm cache and return error", diskURI, nodeName) - klog.Warningf("%v", err) + klog.Warningf("volume %q is actually attached to current node %q, invalidate vm cache and return error", diskURI, nodeName) // update VM(invalidate vm cache) if errUpdate := c.UpdateVM(nodeName); errUpdate != nil { return -1, errUpdate } - return -1, err + lun, _, err := c.GetDiskLun(diskName, diskURI, nodeName) + return lun, err } attachErr := fmt.Sprintf( @@ -219,7 +222,13 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri } c.lockMap.LockEntry(node) - defer c.lockMap.UnlockEntry(node) + unlock := false + defer func() { + if !unlock { + c.lockMap.UnlockEntry(node) + } + }() + diskMap, err := c.cleanAttachDiskRequests(node) if err != nil { return -1, err @@ -241,7 +250,28 @@ func (c *controllerCommon) AttachDisk(isManagedDisk bool, diskName, diskURI stri } c.diskStateMap.Store(disk, "attaching") defer c.diskStateMap.Delete(disk) - return lun, vmset.AttachDisk(nodeName, diskMap) + future, err := vmset.AttachDisk(nodeName, diskMap) + if err != nil { + return -1, err + } + + if c.diskOpRateLimiter.TryAccept() { + // unlock and wait for attach disk complete + unlock = true + c.lockMap.UnlockEntry(node) + } else { + klog.Warningf("azureDisk - switch to batch operation since disk operation is rate limited, current QPS: %f", c.diskOpRateLimiter.QPS()) + } + + ctx, cancel := getContextWithCancel() + defer cancel() + + resourceGroup, err := getResourceGroupFromDiskURI(diskURI) + if err != nil { + return -1, err + } + + return lun, vmset.WaitForUpdateResult(ctx, future, resourceGroup, "attach_disk") } func (c *controllerCommon) insertAttachDiskRequest(diskURI, nodeName string, options *AttachDiskOptions) error { @@ -289,7 +319,9 @@ func (c *controllerCommon) cleanAttachDiskRequests(nodeName string) (map[string] // DetachDisk detaches a disk from VM func (c *controllerCommon) DetachDisk(diskName, diskURI string, nodeName types.NodeName) error { - _, err := c.cloud.InstanceID(context.TODO(), nodeName) + ctx, cancel := getContextWithCancel() + defer cancel() + _, err := c.cloud.InstanceID(ctx, nodeName) if err != nil { if errors.Is(err, cloudprovider.InstanceNotFound) { // if host doesn't exist, no need to detach @@ -308,7 +340,12 @@ func (c *controllerCommon) DetachDisk(diskName, diskURI string, nodeName types.N } c.lockMap.LockEntry(node) - defer c.lockMap.UnlockEntry(node) + unlock := false + defer func() { + if !unlock { + c.lockMap.UnlockEntry(node) + } + }() diskMap, err := c.cleanDetachDiskRequests(node) if err != nil { return err @@ -322,7 +359,8 @@ func (c *controllerCommon) DetachDisk(diskName, diskURI string, nodeName types.N } c.diskStateMap.Store(disk, "detaching") defer c.diskStateMap.Delete(disk) - if err = vmset.DetachDisk(nodeName, diskMap); err != nil { + future, err := vmset.DetachDisk(nodeName, diskMap) + if err != nil { if isInstanceNotFoundError(err) { // if host doesn't exist, no need to detach klog.Warningf("azureDisk - got InstanceNotFoundError(%v), DetachDisk(%s) will assume disk is already detached", @@ -332,6 +370,22 @@ func (c *controllerCommon) DetachDisk(diskName, diskURI string, nodeName types.N klog.Errorf("azureDisk - detach disk(%s, %s) failed, err: %v", diskName, diskURI, err) return err } + resourceGroup, err := getResourceGroupFromDiskURI(diskURI) + if err != nil { + return err + } + if c.diskOpRateLimiter.TryAccept() { + // unlock and wait for attach disk complete + unlock = true + c.lockMap.UnlockEntry(node) + } else { + klog.Warningf("azureDisk - switch to batch operation since disk operation is rate limited, current QPS: %f", c.diskOpRateLimiter.QPS()) + } + + if err := vmset.WaitForUpdateResult(ctx, future, resourceGroup, "detach_disk"); err != nil { + klog.Errorf("azureDisk - detach disk(%s, %s) failed with error: %v", diskName, diskURI, err) + return err + } } klog.V(2).Infof("azureDisk - detach disk(%s, %s) succeeded", diskName, diskURI) diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_standard.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_standard.go index 225ccfd136..b0d6a7910b 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_standard.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_standard.go @@ -17,10 +17,12 @@ limitations under the License. package provider import ( + "context" "net/http" "strings" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-12-01/compute" + "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/to" "k8s.io/apimachinery/pkg/types" @@ -31,16 +33,16 @@ import ( ) // AttachDisk attaches a disk to vm -func (as *availabilitySet) AttachDisk(nodeName types.NodeName, diskMap map[string]*AttachDiskOptions) error { +func (as *availabilitySet) AttachDisk(nodeName types.NodeName, diskMap map[string]*AttachDiskOptions) (*azure.Future, error) { vm, err := as.getVirtualMachine(nodeName, azcache.CacheReadTypeDefault) if err != nil { - return err + return nil, err } vmName := mapNodeNameToVMName(nodeName) nodeResourceGroup, err := as.GetNodeResourceGroup(vmName) if err != nil { - return err + return nil, err } disks := make([]compute.DataDisk, len(*vm.StorageProfile.DataDisks)) @@ -114,37 +116,45 @@ func (as *availabilitySet) AttachDisk(nodeName types.NodeName, diskMap map[strin _ = as.cloud.vmCache.Delete(vmName) }() - rerr := as.VirtualMachinesClient.Update(ctx, nodeResourceGroup, vmName, newVM, "attach_disk") + future, rerr := as.VirtualMachinesClient.UpdateAsync(ctx, nodeResourceGroup, vmName, newVM, "attach_disk") if rerr != nil { klog.Errorf("azureDisk - attach disk list(%s) on rg(%s) vm(%s) failed, err: %v", diskMap, nodeResourceGroup, vmName, rerr) if rerr.HTTPStatusCode == http.StatusNotFound { klog.Errorf("azureDisk - begin to filterNonExistingDisks(%v) on rg(%s) vm(%s)", diskMap, nodeResourceGroup, vmName) disks := as.filterNonExistingDisks(ctx, *newVM.VirtualMachineProperties.StorageProfile.DataDisks) newVM.VirtualMachineProperties.StorageProfile.DataDisks = &disks - rerr = as.VirtualMachinesClient.Update(ctx, nodeResourceGroup, vmName, newVM, "attach_disk") + future, rerr = as.VirtualMachinesClient.UpdateAsync(ctx, nodeResourceGroup, vmName, newVM, "attach_disk") } } klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk list(%s) returned with %v", nodeResourceGroup, vmName, diskMap, rerr) if rerr != nil { + return future, rerr.Error() + } + return future, nil +} + +// WaitForUpdateResult waits for the response of the update request +func (as *availabilitySet) WaitForUpdateResult(ctx context.Context, future *azure.Future, resourceGroupName, source string) error { + if rerr := as.VirtualMachinesClient.WaitForUpdateResult(ctx, future, resourceGroupName, source); rerr != nil { return rerr.Error() } return nil } // DetachDisk detaches a disk from VM -func (as *availabilitySet) DetachDisk(nodeName types.NodeName, diskMap map[string]string) error { +func (as *availabilitySet) DetachDisk(nodeName types.NodeName, diskMap map[string]string) (*azure.Future, error) { vm, err := as.getVirtualMachine(nodeName, azcache.CacheReadTypeDefault) if err != nil { // if host doesn't exist, no need to detach klog.Warningf("azureDisk - cannot find node %s, skip detaching disk list(%s)", nodeName, diskMap) - return nil + return nil, nil } vmName := mapNodeNameToVMName(nodeName) nodeResourceGroup, err := as.GetNodeResourceGroup(vmName) if err != nil { - return err + return nil, err } disks := make([]compute.DataDisk, len(*vm.StorageProfile.DataDisks)) @@ -196,22 +206,22 @@ func (as *availabilitySet) DetachDisk(nodeName types.NodeName, diskMap map[strin _ = as.cloud.vmCache.Delete(vmName) }() - rerr := as.VirtualMachinesClient.Update(ctx, nodeResourceGroup, vmName, newVM, "detach_disk") + future, rerr := as.VirtualMachinesClient.UpdateAsync(ctx, nodeResourceGroup, vmName, newVM, "detach_disk") if rerr != nil { klog.Errorf("azureDisk - detach disk list(%s) on rg(%s) vm(%s) failed, err: %v", diskMap, nodeResourceGroup, vmName, rerr) if rerr.HTTPStatusCode == http.StatusNotFound { klog.Errorf("azureDisk - begin to filterNonExistingDisks(%v) on rg(%s) vm(%s)", diskMap, nodeResourceGroup, vmName) disks := as.filterNonExistingDisks(ctx, *vm.StorageProfile.DataDisks) newVM.VirtualMachineProperties.StorageProfile.DataDisks = &disks - rerr = as.VirtualMachinesClient.Update(ctx, nodeResourceGroup, vmName, newVM, "detach_disk") + future, rerr = as.VirtualMachinesClient.UpdateAsync(ctx, nodeResourceGroup, vmName, newVM, "detach_disk") } } klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk list(%s) returned with %v", nodeResourceGroup, vmName, diskMap, rerr) if rerr != nil { - return rerr.Error() + return future, rerr.Error() } - return nil + return future, nil } // UpdateVM updates a vm diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_vmss.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_vmss.go index 31c262ac98..2dd73ed4f5 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_vmss.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_controller_vmss.go @@ -17,10 +17,12 @@ limitations under the License. package provider import ( + "context" "net/http" "strings" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-12-01/compute" + "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/to" "k8s.io/apimachinery/pkg/types" @@ -31,16 +33,16 @@ import ( ) // AttachDisk attaches a disk to vm -func (ss *ScaleSet) AttachDisk(nodeName types.NodeName, diskMap map[string]*AttachDiskOptions) error { +func (ss *ScaleSet) AttachDisk(nodeName types.NodeName, diskMap map[string]*AttachDiskOptions) (*azure.Future, error) { vmName := mapNodeNameToVMName(nodeName) ssName, instanceID, vm, err := ss.getVmssVM(vmName, azcache.CacheReadTypeDefault) if err != nil { - return err + return nil, err } nodeResourceGroup, err := ss.GetNodeResourceGroup(vmName) if err != nil { - return err + return nil, err } disks := []compute.DataDisk{} @@ -118,35 +120,43 @@ func (ss *ScaleSet) AttachDisk(nodeName types.NodeName, diskMap map[string]*Atta }() klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk list(%s)", nodeResourceGroup, nodeName, diskMap) - rerr := ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM, "attach_disk") + future, rerr := ss.VirtualMachineScaleSetVMsClient.UpdateAsync(ctx, nodeResourceGroup, ssName, instanceID, newVM, "attach_disk") if rerr != nil { klog.Errorf("azureDisk - attach disk list(%s) on rg(%s) vm(%s) failed, err: %v", diskMap, nodeResourceGroup, nodeName, rerr) if rerr.HTTPStatusCode == http.StatusNotFound { klog.Errorf("azureDisk - begin to filterNonExistingDisks(%v) on rg(%s) vm(%s)", diskMap, nodeResourceGroup, nodeName) disks := ss.filterNonExistingDisks(ctx, *newVM.VirtualMachineScaleSetVMProperties.StorageProfile.DataDisks) newVM.VirtualMachineScaleSetVMProperties.StorageProfile.DataDisks = &disks - rerr = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM, "attach_disk") + future, rerr = ss.VirtualMachineScaleSetVMsClient.UpdateAsync(ctx, nodeResourceGroup, ssName, instanceID, newVM, "attach_disk") } } klog.V(2).Infof("azureDisk - update(%s): vm(%s) - attach disk list(%s, %s) returned with %v", nodeResourceGroup, nodeName, diskMap, rerr) if rerr != nil { + return future, rerr.Error() + } + return future, nil +} + +// WaitForUpdateResult waits for the response of the update request +func (ss *ScaleSet) WaitForUpdateResult(ctx context.Context, future *azure.Future, resourceGroupName, source string) error { + if rerr := ss.VirtualMachineScaleSetVMsClient.WaitForUpdateResult(ctx, future, resourceGroupName, source); rerr != nil { return rerr.Error() } return nil } // DetachDisk detaches a disk from VM -func (ss *ScaleSet) DetachDisk(nodeName types.NodeName, diskMap map[string]string) error { +func (ss *ScaleSet) DetachDisk(nodeName types.NodeName, diskMap map[string]string) (*azure.Future, error) { vmName := mapNodeNameToVMName(nodeName) ssName, instanceID, vm, err := ss.getVmssVM(vmName, azcache.CacheReadTypeDefault) if err != nil { - return err + return nil, err } nodeResourceGroup, err := ss.GetNodeResourceGroup(vmName) if err != nil { - return err + return nil, err } disks := []compute.DataDisk{} @@ -201,22 +211,22 @@ func (ss *ScaleSet) DetachDisk(nodeName types.NodeName, diskMap map[string]strin }() klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk list(%s)", nodeResourceGroup, nodeName, diskMap) - rerr := ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM, "detach_disk") + future, rerr := ss.VirtualMachineScaleSetVMsClient.UpdateAsync(ctx, nodeResourceGroup, ssName, instanceID, newVM, "detach_disk") if rerr != nil { klog.Errorf("azureDisk - detach disk list(%s) on rg(%s) vm(%s) failed, err: %v", diskMap, nodeResourceGroup, nodeName, rerr) if rerr.HTTPStatusCode == http.StatusNotFound { klog.Errorf("azureDisk - begin to filterNonExistingDisks(%v) on rg(%s) vm(%s)", diskMap, nodeResourceGroup, nodeName) disks := ss.filterNonExistingDisks(ctx, *newVM.VirtualMachineScaleSetVMProperties.StorageProfile.DataDisks) newVM.VirtualMachineScaleSetVMProperties.StorageProfile.DataDisks = &disks - rerr = ss.VirtualMachineScaleSetVMsClient.Update(ctx, nodeResourceGroup, ssName, instanceID, newVM, "detach_disk") + future, rerr = ss.VirtualMachineScaleSetVMsClient.UpdateAsync(ctx, nodeResourceGroup, ssName, instanceID, newVM, "detach_disk") } } klog.V(2).Infof("azureDisk - update(%s): vm(%s) - detach disk(%v) returned with %v", nodeResourceGroup, nodeName, diskMap, rerr) if rerr != nil { - return rerr.Error() + return future, rerr.Error() } - return nil + return future, nil } // UpdateVM updates a vm diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_fakes.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_fakes.go index 7232283564..f9cc308669 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_fakes.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_fakes.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/routeclient/mockrouteclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/routetableclient/mockroutetableclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/securitygroupclient/mocksecuritygroupclient" + "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/snapshotclient/mocksnapshotclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/subnetclient/mocksubnetclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmclient/mockvmclient" "sigs.k8s.io/cloud-provider-azure/pkg/azureclients/vmssclient/mockvmssclient" @@ -88,6 +89,7 @@ func GetTestCloud(ctrl *gomock.Controller) (az *Cloud) { eventRecorder: &record.FakeRecorder{}, } az.DisksClient = mockdiskclient.NewMockInterface(ctrl) + az.SnapshotsClient = mocksnapshotclient.NewMockInterface(ctrl) az.InterfacesClient = mockinterfaceclient.NewMockInterface(ctrl) az.LoadBalancerClient = mockloadbalancerclient.NewMockInterface(ctrl) az.PublicIPAddressesClient = mockpublicipclient.NewMockInterface(ctrl) diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_instances.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_instances.go index ecaab4b5fb..6f97744b9a 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_instances.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_instances.go @@ -28,6 +28,7 @@ import ( cloudprovider "k8s.io/cloud-provider" "k8s.io/klog/v2" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-12-01/compute" azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache" "sigs.k8s.io/cloud-provider-azure/pkg/consts" ) @@ -218,7 +219,7 @@ func (az *Cloud) InstanceExists(ctx context.Context, node *v1.Node) (bool, error providerID := node.Spec.ProviderID if providerID == "" { var err error - providerID, err = az.getNodeProviderIDByNodeName(ctx, types.NodeName(node.Name)) + providerID, err = cloudprovider.GetInstanceProviderID(ctx, az, types.NodeName(node.Name)) if err != nil { klog.Errorf("InstanceExists: failed to get the provider ID by node name %s: %v", node.Name, err) return false, err @@ -253,10 +254,22 @@ func (az *Cloud) InstanceShutdownByProviderID(ctx context.Context, providerID st return false, err } - klog.V(5).Infof("InstanceShutdownByProviderID gets power status %q for node %q", powerStatus, nodeName) + klog.V(3).Infof("InstanceShutdownByProviderID gets power status %q for node %q", powerStatus, nodeName) + + provisioningState, err := az.VMSet.GetProvisioningStateByNodeName(string(nodeName)) + if err != nil { + // Returns false, so the controller manager will continue to check InstanceExistsByProviderID(). + if errors.Is(err, cloudprovider.InstanceNotFound) { + return false, nil + } + + return false, err + } + klog.V(3).Infof("InstanceShutdownByProviderID gets provisioning state %q for node %q", provisioningState, nodeName) status := strings.ToLower(powerStatus) - return status == vmPowerStateStopped || status == vmPowerStateDeallocated || status == vmPowerStateDeallocating, nil + provisioningSucceeded := strings.EqualFold(strings.ToLower(provisioningState), strings.ToLower(string(compute.ProvisioningStateSucceeded))) + return provisioningSucceeded && (status == vmPowerStateStopped || status == vmPowerStateDeallocated || status == vmPowerStateDeallocating), nil } // InstanceShutdown returns true if the instance is shutdown according to the cloud provider. @@ -268,8 +281,13 @@ func (az *Cloud) InstanceShutdown(ctx context.Context, node *v1.Node) (bool, err providerID := node.Spec.ProviderID if providerID == "" { var err error - providerID, err = az.getNodeProviderIDByNodeName(ctx, types.NodeName(node.Name)) + providerID, err = cloudprovider.GetInstanceProviderID(ctx, az, types.NodeName(node.Name)) if err != nil { + // Returns false, so the controller manager will continue to check InstanceExistsByProviderID(). + if strings.Contains(err.Error(), cloudprovider.InstanceNotFound.Error()) { + return false, nil + } + klog.Errorf("InstanceShutdown: failed to get the provider ID by node name %s: %v", node.Name, err) return false, err } @@ -344,15 +362,6 @@ func (az *Cloud) InstanceID(ctx context.Context, name types.NodeName) (string, e return az.VMSet.GetInstanceIDByNodeName(nodeName) } -func (az *Cloud) getNodeProviderIDByNodeName(ctx context.Context, name types.NodeName) (string, error) { - providerID, err := cloudprovider.GetInstanceProviderID(ctx, az, name) - if err != nil { - return "", fmt.Errorf("failed to get the provider ID of the node %s: %w", string(name), err) - } - - return providerID, nil -} - func (az *Cloud) getLocalInstanceProviderID(metadata *InstanceMetadata, nodeName string) (string, error) { // Get resource group name and subscription ID. resourceGroup := strings.ToLower(metadata.Compute.ResourceGroup) @@ -469,7 +478,7 @@ func (az *Cloud) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudpro if node.Spec.ProviderID != "" { meta.ProviderID = node.Spec.ProviderID } else { - providerID, err := az.getNodeProviderIDByNodeName(ctx, types.NodeName(node.Name)) + providerID, err := cloudprovider.GetInstanceProviderID(ctx, az, types.NodeName(node.Name)) if err != nil { klog.Errorf("InstanceMetadata: failed to get the provider ID by node name %s: %v", node.Name, err) return nil, err diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go index b940ff8078..151505e31e 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_loadbalancer.go @@ -397,20 +397,20 @@ func (az *Cloud) cleanOrphanedLoadBalancer(lb *network.LoadBalancer, service *v1 if deleteErr != nil { klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): failed to DeleteLB: %v", lbName, serviceName, clusterName, deleteErr) - rgName, vmssName, parseErr := retry.GetVMSSMetadataByRawError(deleteErr.Error()) + rgName, vmssName, parseErr := retry.GetVMSSMetadataByRawError(deleteErr) if parseErr != nil { klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): failed to parse error: %v", lbName, serviceName, clusterName, parseErr) - return deleteErr + return deleteErr.Error() } if rgName == "" || vmssName == "" { klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): empty rgName or vmssName", lbName, serviceName, clusterName) - return deleteErr + return deleteErr.Error() } // if we reach here, it means the VM couldn't be deleted because it is being referenced by a VMSS if _, ok := az.VMSet.(*ScaleSet); !ok { klog.Warningf("cleanOrphanedLoadBalancer(%s, %s, %s): unexpected VMSet type, expected VMSS", lbName, serviceName, clusterName) - return deleteErr + return deleteErr.Error() } if !strings.EqualFold(rgName, az.ResourceGroup) { @@ -427,7 +427,7 @@ func (az *Cloud) cleanOrphanedLoadBalancer(lb *network.LoadBalancer, service *v1 deleteErr := az.DeleteLB(service, lbName) if deleteErr != nil { klog.Errorf("cleanOrphanedLoadBalancer(%s, %s, %s): failed delete lb for the second time, stop retrying: %v", lbName, serviceName, clusterName, deleteErr) - return deleteErr + return deleteErr.Error() } } klog.V(10).Infof("cleanOrphanedLoadBalancer(%s, %s, %s): az.DeleteLB finished", lbName, serviceName, clusterName) @@ -444,9 +444,9 @@ func (az *Cloud) safeDeleteLoadBalancer(lb network.LoadBalancer, clusterName, vm } klog.V(2).Infof("deleteDedicatedLoadBalancer: deleting LB %s because the corresponding vmSet is supposed to be in the primary SLB", to.String(lb.Name)) - err = az.DeleteLB(service, to.String(lb.Name)) - if err != nil { - return fmt.Errorf("deleteDedicatedLoadBalancer : failed to DeleteLB: %w", err) + rerr := az.DeleteLB(service, to.String(lb.Name)) + if rerr != nil { + return fmt.Errorf("deleteDedicatedLoadBalancer : failed to DeleteLB: %w", rerr.Error()) } _ = az.lbCache.Delete(to.String(lb.Name)) @@ -1031,6 +1031,7 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai pip.Name = to.StringPtr(pipName) pip.Location = to.StringPtr(az.Location) if az.HasExtendedLocation() { + klog.V(2).Infof("Using extended location with name %s, and type %s for PIP", az.ExtendedLocationName, az.ExtendedLocationType) pip.ExtendedLocation = &network.ExtendedLocation{ Name: &az.ExtendedLocationName, Type: getExtendedLocationTypeFromString(az.ExtendedLocationType), @@ -1053,13 +1054,16 @@ func (az *Cloud) ensurePublicIPExists(service *v1.Service, pipName string, domai Name: network.PublicIPAddressSkuNameStandard, } - // only add zone information for the new standard pips - zones, err := az.getRegionZonesBackoff(to.String(pip.Location)) - if err != nil { - return nil, err - } - if len(zones) > 0 { - pip.Zones = &zones + // skip adding zone info since edge zones doesn't support multiple availability zones. + if !az.HasExtendedLocation() { + // only add zone information for the new standard pips + zones, err := az.getRegionZonesBackoff(to.String(pip.Location)) + if err != nil { + return nil, err + } + if len(zones) > 0 { + pip.Zones = &zones + } } } klog.V(2).Infof("ensurePublicIPExists for service(%s): pip(%s) - creating", serviceName, *pip.Name) @@ -1782,13 +1786,6 @@ func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Serv // construct FrontendIPConfigurationPropertiesFormat var fipConfigurationProperties *network.FrontendIPConfigurationPropertiesFormat if isInternal { - // azure does not support ILB for IPv6 yet. - // TODO: remove this check when ILB supports IPv6 *and* the SDK - // have been rev'ed to 2019* version - if utilnet.IsIPv6String(service.Spec.ClusterIP) { - return nil, false, fmt.Errorf("ensure(%s): lb(%s) - internal load balancers does not support IPv6", serviceName, lbName) - } - subnetName := subnet(service) if subnetName == nil { subnetName = &az.SubnetName @@ -1806,6 +1803,10 @@ func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Serv Subnet: &subnet, } + if utilnet.IsIPv6String(service.Spec.ClusterIP) { + configProperties.PrivateIPAddressVersion = network.IPVersionIPv6 + } + loadBalancerIP := service.Spec.LoadBalancerIP if loadBalancerIP != "" { configProperties.PrivateIPAllocationMethod = network.IPAllocationMethodStatic @@ -1837,13 +1838,13 @@ func (az *Cloud) reconcileFrontendIPConfigs(clusterName string, service *v1.Serv FrontendIPConfigurationPropertiesFormat: fipConfigurationProperties, } - // only add zone information for new internal frontend IP configurations + // only add zone information for new internal frontend IP configurations for standard load balancer not deployed to an edge zone. location := az.Location zones, err := az.getRegionZonesBackoff(location) if err != nil { return nil, false, err } - if isInternal && az.useStandardLoadBalancer() && len(zones) > 0 { + if isInternal && az.useStandardLoadBalancer() && len(zones) > 0 && !az.HasExtendedLocation() { newConfig.Zones = &zones } newConfigs = append(newConfigs, newConfig) @@ -2333,18 +2334,18 @@ func (az *Cloud) reconcileSecurityRules(sg network.SecurityGroup, service *v1.Se sharedRuleName := az.getSecurityRuleName(service, port, sourceAddressPrefix) sharedIndex, sharedRule, sharedRuleFound := findSecurityRuleByName(updatedRules, sharedRuleName) if !sharedRuleFound { - klog.V(4).Infof("Expected to find shared rule %s for service %s being deleted, but did not", sharedRuleName, service.Name) - return false, nil, fmt.Errorf("expected to find shared rule %s for service %s being deleted, but did not", sharedRuleName, service.Name) + klog.V(4).Infof("Didn't find shared rule %s for service %s", sharedRuleName, service.Name) + continue } if sharedRule.DestinationAddressPrefixes == nil { - klog.V(4).Infof("Expected to have array of destinations in shared rule for service %s being deleted, but did not", service.Name) - return false, nil, fmt.Errorf("expected to have array of destinations in shared rule for service %s being deleted, but did not", service.Name) + klog.V(4).Infof("Didn't find DestinationAddressPrefixes in shared rule for service %s", service.Name) + continue } existingPrefixes := *sharedRule.DestinationAddressPrefixes for _, destinationIPAddress := range destinationIPAddresses { addressIndex, found := findIndex(existingPrefixes, destinationIPAddress) if !found { - klog.Warningf("Expected to find destination address %v in shared rule %s for service %s being deleted, but did not", destinationIPAddress, sharedRuleName, service.Name) + klog.Warningf("Didn't find destination address %v in shared rule %s for service %s", destinationIPAddress, sharedRuleName, service.Name) continue } if len(existingPrefixes) == 1 { @@ -2967,7 +2968,7 @@ func findSecurityRule(rules []network.SecurityRule, rule network.SecurityRule) b if !strings.EqualFold(to.String(existingRule.Name), to.String(rule.Name)) { continue } - if existingRule.Protocol != rule.Protocol { + if !strings.EqualFold(string(existingRule.Protocol), string(rule.Protocol)) { continue } if !strings.EqualFold(to.String(existingRule.SourcePortRange), to.String(rule.SourcePortRange)) { @@ -2987,10 +2988,10 @@ func findSecurityRule(rules []network.SecurityRule, rule network.SecurityRule) b continue } } - if existingRule.Access != rule.Access { + if !strings.EqualFold(string(existingRule.Access), string(rule.Access)) { continue } - if existingRule.Direction != rule.Direction { + if !strings.EqualFold(string(existingRule.Direction), string(rule.Direction)) { continue } return true diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_ratelimit.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_ratelimit.go index 94e6acfb91..b9f9206471 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_ratelimit.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_ratelimit.go @@ -21,6 +21,11 @@ import ( "sigs.k8s.io/cloud-provider-azure/pkg/consts" ) +const ( + defaultAtachDetachDiskQPS = 6.0 + defaultAtachDetachDiskBucket = 10 +) + // CloudProviderRateLimitConfig indicates the rate limit config for each clients. type CloudProviderRateLimitConfig struct { // The default rate limit config options. @@ -41,6 +46,7 @@ type CloudProviderRateLimitConfig struct { VirtualMachineScaleSetRateLimit *azclients.RateLimitConfig `json:"virtualMachineScaleSetRateLimit,omitempty" yaml:"virtualMachineScaleSetRateLimit,omitempty"` VirtualMachineSizeRateLimit *azclients.RateLimitConfig `json:"virtualMachineSizesRateLimit,omitempty" yaml:"virtualMachineSizesRateLimit,omitempty"` AvailabilitySetRateLimit *azclients.RateLimitConfig `json:"availabilitySetRateLimit,omitempty" yaml:"availabilitySetRateLimit,omitempty"` + AttachDetachDiskRateLimit *azclients.RateLimitConfig `json:"attachDetachDiskRateLimit,omitempty" yaml:"attachDetachDiskRateLimit,omitempty"` } // InitializeCloudProviderRateLimitConfig initializes rate limit configs. @@ -78,6 +84,13 @@ func InitializeCloudProviderRateLimitConfig(config *CloudProviderRateLimitConfig config.VirtualMachineScaleSetRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.VirtualMachineScaleSetRateLimit) config.VirtualMachineSizeRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.VirtualMachineSizeRateLimit) config.AvailabilitySetRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.AvailabilitySetRateLimit) + + atachDetachDiskRateLimitConfig := azclients.RateLimitConfig{ + CloudProviderRateLimit: true, + CloudProviderRateLimitQPSWrite: defaultAtachDetachDiskQPS, + CloudProviderRateLimitBucketWrite: defaultAtachDetachDiskBucket, + } + config.AttachDetachDiskRateLimit = overrideDefaultRateLimitConfig(&atachDetachDiskRateLimitConfig, config.AttachDetachDiskRateLimit) } // overrideDefaultRateLimitConfig overrides the default CloudProviderRateLimitConfig. diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_standard.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_standard.go index a995e33bc7..5018e48541 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_standard.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_standard.go @@ -368,8 +368,6 @@ func (az *Cloud) serviceOwnsFrontendIP(fip network.FrontendIPConfiguration, serv return true, isPrimaryService, nil } klog.V(4).Infof("serviceOwnsFrontendIP: the public IP with ID %s is being referenced by other service with public IP address %s", *pip.ID, *pip.IPAddress) - - return false, isPrimaryService, nil } return false, isPrimaryService, nil @@ -549,6 +547,20 @@ func (as *availabilitySet) GetPowerStatusByNodeName(name string) (powerState str return vmPowerStateStopped, nil } +// GetProvisioningStateByNodeName returns the provisioningState for the specified node. +func (as *availabilitySet) GetProvisioningStateByNodeName(name string) (provisioningState string, err error) { + vm, err := as.getVirtualMachine(types.NodeName(name), azcache.CacheReadTypeDefault) + if err != nil { + return provisioningState, err + } + + if vm.VirtualMachineProperties == nil || vm.VirtualMachineProperties.ProvisioningState == nil { + return provisioningState, nil + } + + return to.String(vm.VirtualMachineProperties.ProvisioningState), nil +} + // GetNodeNameByProviderID gets the node name by provider ID. func (as *availabilitySet) GetNodeNameByProviderID(providerID string) (types.NodeName, error) { // NodeName is part of providerID for standard instances. diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go index 00e6a424a5..cfc981fa04 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_storageaccount.go @@ -123,6 +123,7 @@ func (az *Cloud) EnsureStorageAccount(accountOptions *AccountOptions, genAccount if accountOptions == nil { return "", "", fmt.Errorf("account options is nil") } + accountName := accountOptions.Name accountType := accountOptions.Type accountKind := accountOptions.Kind @@ -212,10 +213,10 @@ func (az *Cloud) EnsureStorageAccount(accountOptions *AccountOptions, genAccount if az.StorageAccountClient == nil { return "", "", fmt.Errorf("StorageAccountClient is nil") } + ctx, cancel := getContextWithCancel() defer cancel() - rerr := az.StorageAccountClient.Create(ctx, resourceGroup, accountName, cp) - if rerr != nil { + if rerr := az.StorageAccountClient.Create(ctx, resourceGroup, accountName, cp); rerr != nil { return "", "", fmt.Errorf("failed to create storage account %s, error: %v", accountName, rerr) } @@ -235,33 +236,37 @@ func (az *Cloud) EnsureStorageAccount(accountOptions *AccountOptions, genAccount } if accountOptions.CreatePrivateEndpoint { + vnetResourceGroup := az.ResourceGroup + if len(az.VnetResourceGroup) > 0 { + vnetResourceGroup = az.VnetResourceGroup + } // Get properties of the storageAccount - storageAccount, err := az.StorageAccountClient.GetProperties(ctx, az.ResourceGroup, accountName) + storageAccount, err := az.StorageAccountClient.GetProperties(ctx, resourceGroup, accountName) if err != nil { - return "", "", fmt.Errorf("Failed to get the properties of storage account(%s), resourceGroup(%s), error: %v", accountName, az.ResourceGroup, err) + return "", "", fmt.Errorf("Failed to get the properties of storage account(%s), resourceGroup(%s), error: %v", accountName, resourceGroup, err) } // Create private endpoint privateEndpointName := accountName + "-pvtendpoint" - if err := az.createPrivateEndpoint(ctx, accountName, storageAccount.ID, privateEndpointName); err != nil { - return "", "", fmt.Errorf("Failed to create private endpoint for storage account(%s), resourceGroup(%s), error: %v", accountName, az.ResourceGroup, err) + if err := az.createPrivateEndpoint(ctx, accountName, storageAccount.ID, privateEndpointName, vnetResourceGroup); err != nil { + return "", "", fmt.Errorf("Failed to create private endpoint for storage account(%s), resourceGroup(%s), error: %v", accountName, vnetResourceGroup, err) } // Create DNS zone - if err := az.createPrivateDNSZone(ctx); err != nil { - return "", "", fmt.Errorf("Failed to create private DNS zone(%s) in resourceGroup(%s), error: %v", PrivateDNSZoneName, az.ResourceGroup, err) + if err := az.createPrivateDNSZone(ctx, vnetResourceGroup); err != nil { + return "", "", fmt.Errorf("Failed to create private DNS zone(%s) in resourceGroup(%s), error: %v", PrivateDNSZoneName, vnetResourceGroup, err) } // Create virtual link to the zone private DNS zone vNetLinkName := accountName + "-vnetlink" - if err := az.createVNetLink(ctx, vNetLinkName); err != nil { - return "", "", fmt.Errorf("Failed to create virtual link for vnet(%s) and DNS Zone(%s) in resourceGroup(%s), error: %v", az.VnetName, PrivateDNSZoneName, az.ResourceGroup, err) + if err := az.createVNetLink(ctx, vNetLinkName, vnetResourceGroup); err != nil { + return "", "", fmt.Errorf("Failed to create virtual link for vnet(%s) and DNS Zone(%s) in resourceGroup(%s), error: %v", az.VnetName, PrivateDNSZoneName, vnetResourceGroup, err) } // Create dns zone group dnsZoneGroupName := accountName + "-dnszonegroup" - if err := az.createPrivateDNSZoneGroup(ctx, dnsZoneGroupName, privateEndpointName); err != nil { - return "", "", fmt.Errorf("Failed to create private DNS zone group - privateEndpoint(%s), vNetName(%s), resourceGroup(%s), error: %v", privateEndpointName, az.VnetName, az.ResourceGroup, err) + if err := az.createPrivateDNSZoneGroup(ctx, dnsZoneGroupName, privateEndpointName, vnetResourceGroup); err != nil { + return "", "", fmt.Errorf("Failed to create private DNS zone group - privateEndpoint(%s), vNetName(%s), resourceGroup(%s), error: %v", privateEndpointName, az.VnetName, vnetResourceGroup, err) } } } @@ -276,17 +281,16 @@ func (az *Cloud) EnsureStorageAccount(accountOptions *AccountOptions, genAccount return accountName, accountKey, nil } -func (az *Cloud) createPrivateEndpoint(ctx context.Context, accountName string, accountID *string, privateEndpointName string) error { +func (az *Cloud) createPrivateEndpoint(ctx context.Context, accountName string, accountID *string, privateEndpointName, vnetResourceGroup string) error { klog.V(2).Infof("Creating private endpoint(%s) for account (%s)", privateEndpointName, accountName) - subnet, rerr := az.SubnetsClient.Get(ctx, az.ResourceGroup, az.VnetName, az.SubnetName, "") - if rerr != nil { - return rerr.Error() - } + subnet, _, err := az.getSubnet(az.VnetName, az.SubnetName) + if err != nil { + return err + } // Disable the private endpoint network policies before creating private endpoint subnet.SubnetPropertiesFormat.PrivateEndpointNetworkPolicies = network.VirtualNetworkPrivateEndpointNetworkPoliciesDisabled - rerr = az.SubnetsClient.CreateOrUpdate(ctx, az.ResourceGroup, az.VnetName, az.SubnetName, subnet) - if rerr != nil { + if rerr := az.SubnetsClient.CreateOrUpdate(ctx, vnetResourceGroup, az.VnetName, az.SubnetName, subnet); rerr != nil { return rerr.Error() } @@ -304,42 +308,39 @@ func (az *Cloud) createPrivateEndpoint(ctx context.Context, accountName string, Location: &az.Location, PrivateEndpointProperties: &network.PrivateEndpointProperties{Subnet: &subnet, PrivateLinkServiceConnections: &privateLinkServiceConnections}, } - if err := az.privateendpointclient.CreateOrUpdate(ctx, az.ResourceGroup, privateEndpointName, privateEndpoint, true); err != nil { - return err - } - return nil + return az.privateendpointclient.CreateOrUpdate(ctx, vnetResourceGroup, privateEndpointName, privateEndpoint, true) } -func (az *Cloud) createPrivateDNSZone(ctx context.Context) error { - klog.V(2).Infof("Creating private dns zone(%s) in resourceGroup (%s)", PrivateDNSZoneName, az.ResourceGroup) +func (az *Cloud) createPrivateDNSZone(ctx context.Context, vnetResourceGroup string) error { + klog.V(2).Infof("Creating private dns zone(%s) in resourceGroup (%s)", PrivateDNSZoneName, vnetResourceGroup) location := LocationGlobal privateDNSZone := privatedns.PrivateZone{Location: &location} - if err := az.privatednsclient.CreateOrUpdate(ctx, az.ResourceGroup, PrivateDNSZoneName, privateDNSZone, true); err != nil { + if err := az.privatednsclient.CreateOrUpdate(ctx, vnetResourceGroup, PrivateDNSZoneName, privateDNSZone, true); err != nil { + if strings.Contains(err.Error(), "exists already") { + klog.V(2).Infof("private dns zone(%s) in resourceGroup (%s) already exists", PrivateDNSZoneName, vnetResourceGroup) + return nil + } return err } return nil } -func (az *Cloud) createVNetLink(ctx context.Context, vNetLinkName string) error { - klog.V(2).Infof("Creating virtual link for vnet(%s) and DNS Zone(%s) in resourceGroup(%s)", vNetLinkName, PrivateDNSZoneName, az.ResourceGroup) +func (az *Cloud) createVNetLink(ctx context.Context, vNetLinkName, vnetResourceGroup string) error { + klog.V(2).Infof("Creating virtual link for vnet(%s) and DNS Zone(%s) in resourceGroup(%s)", vNetLinkName, PrivateDNSZoneName, vnetResourceGroup) location := LocationGlobal - vnetID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s", az.SubscriptionID, az.ResourceGroup, az.VnetName) - registrationEnabled := false + vnetID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s", az.SubscriptionID, vnetResourceGroup, az.VnetName) parameters := privatedns.VirtualNetworkLink{ Location: &location, VirtualNetworkLinkProperties: &privatedns.VirtualNetworkLinkProperties{ VirtualNetwork: &privatedns.SubResource{ID: &vnetID}, - RegistrationEnabled: ®istrationEnabled}, - } - if err := az.virtualNetworkLinksClient.CreateOrUpdate(ctx, az.ResourceGroup, PrivateDNSZoneName, vNetLinkName, parameters, false); err != nil { - return err + RegistrationEnabled: to.BoolPtr(true)}, } - return nil + return az.virtualNetworkLinksClient.CreateOrUpdate(ctx, vnetResourceGroup, PrivateDNSZoneName, vNetLinkName, parameters, false) } -func (az *Cloud) createPrivateDNSZoneGroup(ctx context.Context, dnsZoneGroupName string, privateEndpointName string) error { - klog.V(2).Infof("Creating private DNS zone group(%s) with privateEndpoint(%s), vNetName(%s), resourceGroup(%s)", dnsZoneGroupName, privateEndpointName, az.VnetName, az.ResourceGroup) - privateDNSZoneID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/privateDnsZones/%s", az.SubscriptionID, az.ResourceGroup, PrivateDNSZoneName) +func (az *Cloud) createPrivateDNSZoneGroup(ctx context.Context, dnsZoneGroupName, privateEndpointName, vnetResourceGroup string) error { + klog.V(2).Infof("Creating private DNS zone group(%s) with privateEndpoint(%s), vNetName(%s), resourceGroup(%s)", dnsZoneGroupName, privateEndpointName, az.VnetName, vnetResourceGroup) + privateDNSZoneID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/privateDnsZones/%s", az.SubscriptionID, vnetResourceGroup, PrivateDNSZoneName) dnsZoneName := PrivateDNSZoneName privateDNSZoneConfig := network.PrivateDNSZoneConfig{ Name: &dnsZoneName, @@ -352,10 +353,7 @@ func (az *Cloud) createPrivateDNSZoneGroup(ctx context.Context, dnsZoneGroupName PrivateDNSZoneConfigs: &privateDNSZoneConfigs, }, } - if err := az.privatednszonegroupclient.CreateOrUpdate(ctx, az.ResourceGroup, privateEndpointName, dnsZoneGroupName, privateDNSZoneGroup, false); err != nil { - return err - } - return nil + return az.privatednszonegroupclient.CreateOrUpdate(ctx, vnetResourceGroup, privateEndpointName, dnsZoneGroupName, privateDNSZoneGroup, false) } // AddStorageAccountTags add tags to storage account @@ -480,8 +478,11 @@ func isEnableNfsV3PropertyEqual(account storage.Account, accountOptions *Account } func isPrivateEndpointAsExpected(account storage.Account, accountOptions *AccountOptions) bool { - if accountOptions.CreatePrivateEndpoint && (account.PrivateEndpointConnections == nil || len(*account.PrivateEndpointConnections) == 0) { - return false + if accountOptions.CreatePrivateEndpoint && account.PrivateEndpointConnections != nil && len(*account.PrivateEndpointConnections) > 0 { + return true } - return true + if !accountOptions.CreatePrivateEndpoint && (account.PrivateEndpointConnections == nil || len(*account.PrivateEndpointConnections) == 0) { + return true + } + return false } diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_vmsets.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_vmsets.go index 5fa635bb54..33232327a8 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_vmsets.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_vmsets.go @@ -17,8 +17,11 @@ limitations under the License. package provider import ( + "context" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-12-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-02-01/network" + "github.com/Azure/go-autorest/autorest/azure" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -30,7 +33,7 @@ import ( // VMSet defines functions all vmsets (including scale set and availability // set) should be implemented. // Don't forget to run the following command to generate the mock client: -// mockgen -source=$GOPATH/src/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_vmsets.go -package=provider VMSet > $GOPATH/src/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_mock_vmsets.go +// mockgen -source=$GOPATH/src/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_vmsets.go -package=provider VMSet > $GOPATH/src/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_mock_vmsets_test.go type VMSet interface { // GetInstanceIDByNodeName gets the cloud provider ID by node name. // It must return ("", cloudprovider.InstanceNotFound) if the instance does @@ -68,18 +71,24 @@ type VMSet interface { EnsureBackendPoolDeletedFromVMSets(vmSetNamesMap map[string]bool, backendPoolID string) error // AttachDisk attaches a disk to vm - AttachDisk(nodeName types.NodeName, diskMap map[string]*AttachDiskOptions) error + AttachDisk(nodeName types.NodeName, diskMap map[string]*AttachDiskOptions) (*azure.Future, error) // DetachDisk detaches a disk from vm - DetachDisk(nodeName types.NodeName, diskMap map[string]string) error + DetachDisk(nodeName types.NodeName, diskMap map[string]string) (*azure.Future, error) + // WaitForUpdateResult waits for the response of the update request + WaitForUpdateResult(ctx context.Context, future *azure.Future, resourceGroupName, source string) error + // GetDataDisks gets a list of data disks attached to the node. - GetDataDisks(nodeName types.NodeName, string azcache.AzureCacheReadType) ([]compute.DataDisk, *string, error) + GetDataDisks(nodeName types.NodeName, crt azcache.AzureCacheReadType) ([]compute.DataDisk, *string, error) // UpdateVM updates a vm UpdateVM(nodeName types.NodeName) error - // GetPowerStatusByNodeName returns the power state of the specified node. + // GetPowerStatusByNodeName returns the powerState for the specified node. GetPowerStatusByNodeName(name string) (string, error) + // GetProvisioningStateByNodeName returns the provisioningState for the specified node. + GetProvisioningStateByNodeName(name string) (string, error) + // GetPrivateIPsByNodeName returns a slice of all private ips assigned to node (ipv6 and ipv4) GetPrivateIPsByNodeName(name string) ([]string, error) diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_vmss.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_vmss.go index 6b9db90cce..4a13707a25 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_vmss.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_vmss.go @@ -243,6 +243,30 @@ func (ss *ScaleSet) GetPowerStatusByNodeName(name string) (powerState string, er return vmPowerStateStopped, nil } +// GetProvisioningStateByNodeName returns the provisioningState for the specified node. +func (ss *ScaleSet) GetProvisioningStateByNodeName(name string) (provisioningState string, err error) { + managedByAS, err := ss.isNodeManagedByAvailabilitySet(name, azcache.CacheReadTypeUnsafe) + if err != nil { + klog.Errorf("Failed to check isNodeManagedByAvailabilitySet: %v", err) + return "", err + } + if managedByAS { + // vm is managed by availability set. + return ss.availabilitySet.GetProvisioningStateByNodeName(name) + } + + _, _, vm, err := ss.getVmssVM(name, azcache.CacheReadTypeDefault) + if err != nil { + return provisioningState, err + } + + if vm.VirtualMachineScaleSetVMProperties == nil || vm.VirtualMachineScaleSetVMProperties.ProvisioningState == nil { + return provisioningState, nil + } + + return to.String(vm.VirtualMachineScaleSetVMProperties.ProvisioningState), nil +} + // getCachedVirtualMachineByInstanceID gets scaleSetVMInfo from cache. // The node must belong to one of scale sets. func (ss *ScaleSet) getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID string, crt azcache.AzureCacheReadType) (*compute.VirtualMachineScaleSetVM, error) { diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_zones.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_zones.go index e2180df605..d4abbf12e4 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_zones.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/azure_zones.go @@ -73,6 +73,13 @@ func (az *Cloud) updateRegionZonesMap(zones map[string][]string) { } func (az *Cloud) getRegionZonesBackoff(region string) ([]string, error) { + if az.isStackCloud() { + // Azure Stack does not support zone at the moment + // https://docs.microsoft.com/en-us/azure-stack/user/azure-stack-network-differences?view=azs-2102 + klog.V(3).Infof("getRegionZonesMapWrapper: Azure Stack does not support Zones at the moment, skipping") + return az.regionZonesMap[region], nil + } + if len(az.regionZonesMap) != 0 { az.refreshZonesLock.RLock() defer az.refreshZonesLock.RUnlock() diff --git a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/doc.go b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/doc.go index 26587698b0..cded93b190 100644 --- a/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/doc.go +++ b/vendor/sigs.k8s.io/cloud-provider-azure/pkg/provider/doc.go @@ -16,4 +16,4 @@ limitations under the License. // Package provider is an implementation of CloudProvider Interface, LoadBalancer // and Instances for Azure. -package provider // import "sigs.k8s.io/cloud-provider-azure/pkg/provider +package provider