Add retries for modified services to google_project_service (#3960) (#…

…7230)

Signed-off-by: Modular Magician <[email protected]>

.changelog/3960.txt

@@ -0,0 +1,6 @@
+```release-note:bug
+serviceusage: added retries when a service is already being modified
+```
+```release-note:bug
+serviceusage: fixed an issue where `bigquery.googleapis.com` was getting enabled as the `bigquery-json.googleapis.com` alias instead, incorrectly. This had no user impact yet, but the alias may go away in the future.
+```

google/error_retry_predicates.go

@@ -190,6 +190,21 @@ func isSqlOperationInProgressError(err error) (bool, string) {
 	return false, ""
 }
 
+// Retry if service usage decides you're activating the same service multiple
+// times. This can happen if a service and a dependent service aren't batched
+// together- eg container.googleapis.com in one request followed by compute.g.c
+// in the next (container relies on compute and implicitly activates it)
+func serviceUsageServiceBeingActivated(err error) (bool, string) {
+	if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 400 {
+		if strings.Contains(gerr.Body, "Another activation or deactivation is in progress") {
+			return false, ""
+		}
+
+		return true, "Waiting for same service activation/deactivation to finish"
+	}
+	return false, ""
+}
+
 // Retry if Monitoring operation returns a 429 with a specific message for
 // concurrent operations.
 func isMonitoringConcurrentEditError(err error) (bool, string) {

google/resource_google_project.go

@@ -574,7 +574,7 @@ func doEnableServicesRequest(services []string, project string, config *Config,
 			op, rerr = config.clientServiceUsage.Services.BatchEnable(name, req).Do()
 		}
 		return handleServiceUsageRetryableError(rerr)
-	}, timeout)
+	}, timeout, serviceUsageServiceBeingActivated)
 	if err != nil {
 		return errwrap.Wrapf("failed to send enable services request: {{err}}", err)
 	}

google/serviceusage_batching.go

@@ -42,7 +42,7 @@ func tryEnableRenamedService(service, altName string, project string, d *schema.
 	// use a short timeout- failures are likely
 
 	log.Printf("[DEBUG] attempting enabling service with user-specified name %s", service)
-	err := enableServiceUsageProjectServices([]string{altName}, project, config, 1*time.Minute)
+	err := enableServiceUsageProjectServices([]string{service}, project, config, 1*time.Minute)
 	if err != nil {
 		log.Printf("[DEBUG] saw error %s. attempting alternate name %v", err, altName)
 		err2 := enableServiceUsageProjectServices([]string{altName}, project, config, 1*time.Minute)