fix: add istio policy before deployment (#3176)

backend/controller/scaling/k8sscaling/deployment_provisioner.go

@@ -237,6 +237,14 @@ func (r *DeploymentProvisioner) handleNewDeployment(ctx context.Context, dep *sc
 		logger.Debugf("Service account %s already exists", name)
 	}
 
+	// Sync the istio policy if applicable
+	if sec, ok := r.IstioSecurity.Get(); ok {
+		err = r.syncIstioPolicy(ctx, sec, name, service, thisDeployment)
+		if err != nil {
+			return err
+		}
+	}
+
 	// Now create the deployment
 
 	logger.Debugf("Creating new kube deployment %s", name)
@@ -271,12 +279,7 @@ func (r *DeploymentProvisioner) handleNewDeployment(ctx context.Context, dep *sc
 	deployment.Spec.Template.ObjectMeta.Labels = addLabel(deployment.Spec.Template.ObjectMeta.Labels, "app", name)
 	deployment.Spec.Template.Spec.ServiceAccountName = name
 	changes, err := r.syncDeployment(ctx, thisImage, deployment, dep)
-	if sec, ok := r.IstioSecurity.Get(); ok {
-		err = r.syncIstioPolicy(ctx, sec, name, service, thisDeployment)
-		if err != nil {
-			return err
-		}
-	}
+
 	if err != nil {
 		return err
 	}

backend/controller/scaling/kube_scaling_integration_test.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/alecthomas/assert/v2"
 	"github.com/alecthomas/atomic"
@@ -54,7 +55,7 @@ func TestKubeScaling(t *testing.T) {
 			go func() {
 				defer func() {
 					if r := recover(); r != nil {
-						failure.Store(fmt.Errorf("panic in verb: %v", r))
+						failure.Store(fmt.Errorf("panic in verb: %v at %v", r, time.Now()))
 					}
 					routineStopped.Done()
 				}()

backend/runner/runner.go

@@ -247,6 +247,7 @@ type Service struct {
 	identity   *identity.Store
 	lock       sync.Mutex
 	deployment atomic.Value[optional.Option[*deployment]]
+	readyTime  atomic.Value[time.Time]
 
 	config           Config
 	controllerClient ftlv1connect.ControllerServiceClient
@@ -363,6 +364,7 @@ func (s *Service) deploy(ctx context.Context) error {
 	}
 
 	dep := s.makeDeployment(cmdCtx, key, deployment)
+	s.readyTime.Store(time.Now().Add(time.Second * 2)) // Istio is a bit flakey, add a small delay for readiness
 	s.deployment.Store(optional.Some(dep))
 	logger.Debugf("Deployed %s", key)
 	context.AfterFunc(ctx, func() {
@@ -525,6 +527,10 @@ func (s *Service) getDeploymentLogger(ctx context.Context, deploymentKey model.D
 
 func (s *Service) healthCheck(writer http.ResponseWriter, request *http.Request) {
 	if s.deployment.Load().Ok() {
+		if s.readyTime.Load().After(time.Now()) {
+			writer.WriteHeader(http.StatusServiceUnavailable)
+			return
+		}
 		writer.WriteHeader(http.StatusOK)
 		return
 	}