Skip to content

Commit

Permalink
[Multicluster] Fix cache options for controller Manager
Browse files Browse the repository at this point in the history
A few issues were introduced by antrea-io#5843 because of changes in the
sigs.k8s.io/controller-runtime interface.

The biggest issue was that the call to ctrl.NewManager was not using the
Options object populated earlier in the setupManagerAndCertController
function. Instead it was creating and using a new, incomplete Options
object.

Fixes antrea-io#6149

Signed-off-by: Antonin Bas <[email protected]>
  • Loading branch information
antoninbas committed Mar 26, 2024
1 parent 91f374b commit f7e8260
Show file tree
Hide file tree
Showing 4 changed files with 37 additions and 34 deletions.
58 changes: 35 additions & 23 deletions multicluster/cmd/multicluster-controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
apiextensionclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
genericoptions "k8s.io/apiserver/pkg/server/options"
clientset "k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -139,6 +138,15 @@ func getWebhookLabel(isLeader bool, controllerNs string) *metav1.LabelSelector {
func setupManagerAndCertController(isLeader bool, o *Options) (manager.Manager, error) {
ctrl.SetLogger(klog.NewKlogr())

podNamespace := env.GetPodNamespace()

var caConfig *certificate.CAConfig
if isLeader {
caConfig = getCaConfig(isLeader, podNamespace)
} else {
caConfig = getCaConfig(isLeader, "")
}

// build up cert controller to manage certificate for MC Controller
k8sConfig := ctrl.GetConfigOrDie()
k8sConfig.QPS = common.ResourceExchangeQPS
Expand All @@ -149,40 +157,48 @@ func setupManagerAndCertController(isLeader bool, o *Options) (manager.Manager,
}

secureServing := genericoptions.NewSecureServingOptions().WithLoopback()
caCertController, err := certificate.ApplyServerCert(o.SelfSignedCert, client, aggregatorClient, apiExtensionClient,
secureServing, getCaConfig(isLeader, o.Namespace))
caCertController, err := certificate.ApplyServerCert(o.SelfSignedCert, client, aggregatorClient, apiExtensionClient, secureServing, caConfig)
if err != nil {
return nil, fmt.Errorf("error applying server cert: %v", err)
}
if err := caCertController.RunOnce(context.TODO()); err != nil {
return nil, err
}

options := o.options
if o.SelfSignedCert {
o.options.Metrics.CertDir = selfSignedCertDir
options.Metrics.CertDir = selfSignedCertDir
o.WebhookConfig.CertDir = selfSignedCertDir
} else {
o.options.Metrics.CertDir = certDir
options.Metrics.CertDir = certDir
o.WebhookConfig.CertDir = certDir
}
o.options.WebhookServer = webhook.NewServer(webhook.Options{
options.WebhookServer = webhook.NewServer(webhook.Options{
Port: *o.WebhookConfig.Port,
Host: o.WebhookConfig.Host,
CertDir: o.WebhookConfig.CertDir,
})

namespaceFieldSelector := fields.SelectorFromSet(fields.Set{"metadata.namespace": env.GetPodNamespace()})
o.options.Cache.DefaultFieldSelector = namespaceFieldSelector
o.options.Cache.ByObject = map[controllerruntimeclient.Object]cache.ByObject{
&mcv1alpha1.Gateway{}: {
Field: namespaceFieldSelector,
},
&mcv1alpha2.ClusterSet{}: {
Field: namespaceFieldSelector,
},
&mcv1alpha1.MemberClusterAnnounce{}: {
Field: namespaceFieldSelector,
},
cacheOptions := &options.Cache
if isLeader {
// For the leader, restrict the cache to the controller's Namespace.
cacheOptions.DefaultNamespaces = map[string]cache.Config{
podNamespace: {},
}
} else {
// For a member, restict the cache to the controller's Namespace for the following objects.
cacheOptions.ByObject = map[controllerruntimeclient.Object]cache.ByObject{
&mcv1alpha1.Gateway{}: {
Namespaces: map[string]cache.Config{
podNamespace: {},
},
},
&mcv1alpha2.ClusterSet{}: {
Namespaces: map[string]cache.Config{
podNamespace: {},
},
},
}
}

// EndpointSlice is enabled in AntreaProxy by default since v1.11, so Antrea MC
Expand All @@ -206,11 +222,7 @@ func setupManagerAndCertController(isLeader bool, o *Options) (manager.Manager,
}
o.ClusterCalimCRDAvailable = clusterClaimCRDAvailable

mgr, err := ctrl.NewManager(k8sConfig, manager.Options{
Scheme: o.options.Scheme,
Metrics: o.options.Metrics,
HealthProbeBindAddress: o.options.HealthProbeBindAddress,
})
mgr, err := ctrl.NewManager(k8sConfig, options)
if err != nil {
return nil, fmt.Errorf("error creating manager: %v", err)
}
Expand Down
2 changes: 0 additions & 2 deletions multicluster/cmd/multicluster-controller/leader.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,7 @@ func newLeaderCommand() *cobra.Command {
}

func runLeader(o *Options) error {
// on the leader we want the reconciler to run for a given Namespace instead of cluster scope
podNamespace := env.GetPodNamespace()
o.Namespace = podNamespace
stopCh := signals.RegisterSignalHandlers()

mgr, err := setupManagerAndCertControllerFunc(true, o)
Expand Down
6 changes: 2 additions & 4 deletions multicluster/cmd/multicluster-controller/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ type Options struct {
// The path of configuration file.
configFile string
SelfSignedCert bool
options ctrl.Options
Namespace string
// options store some base controller Manager options (initialized from the provided config).
options ctrl.Options
// The Service ClusterIP range used in the member cluster.
ServiceCIDR string
// PodCIDRs is the Pod IP address CIDRs of the member cluster.
Expand Down Expand Up @@ -68,14 +68,12 @@ func newOptions() *Options {
func (o *Options) complete(args []string) error {
var err error
o.setDefaults()
options := ctrl.Options{Scheme: scheme}
ctrlConfig := &mcsv1alpha1.MultiClusterConfig{}
if len(o.configFile) > 0 {
klog.InfoS("Loading config", "file", o.configFile)
if err = o.loadConfigFromFile(ctrlConfig); err != nil {
return err
}
o.options = options
if ctrlConfig.ServiceCIDR != "" {
if _, _, err := net.ParseCIDR(ctrlConfig.ServiceCIDR); err != nil {
return fmt.Errorf("failed to parse serviceCIDR, invalid CIDR string %s", ctrlConfig.ServiceCIDR)
Expand Down
5 changes: 0 additions & 5 deletions multicluster/cmd/multicluster-controller/options_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"testing"

"github.com/stretchr/testify/assert"
ctrl "sigs.k8s.io/controller-runtime"
)

func TestComplete(t *testing.T) {
Expand All @@ -34,7 +33,6 @@ func TestComplete(t *testing.T) {
o: Options{
configFile: "./testdata/antrea-mc-config-with-valid-podcidrs.yml",
SelfSignedCert: false,
options: ctrl.Options{},
ServiceCIDR: "",
PodCIDRs: nil,
GatewayIPPrecedence: "",
Expand All @@ -47,7 +45,6 @@ func TestComplete(t *testing.T) {
o: Options{
configFile: "./testdata/antrea-mc-config-with-empty-podcidrs.yml",
SelfSignedCert: false,
options: ctrl.Options{},
ServiceCIDR: "",
PodCIDRs: nil,
GatewayIPPrecedence: "",
Expand All @@ -60,7 +57,6 @@ func TestComplete(t *testing.T) {
o: Options{
configFile: "./testdata/antrea-mc-config-with-invalid-podcidrs.yml",
SelfSignedCert: false,
options: ctrl.Options{},
ServiceCIDR: "10.100.0.0/16",
PodCIDRs: nil,
GatewayIPPrecedence: "",
Expand All @@ -73,7 +69,6 @@ func TestComplete(t *testing.T) {
o: Options{
configFile: "./testdata/antrea-mc-config-with-invalid-endpointiptype.yml",
SelfSignedCert: false,
options: ctrl.Options{},
ServiceCIDR: "10.100.0.0/16",
PodCIDRs: nil,
GatewayIPPrecedence: "",
Expand Down

0 comments on commit f7e8260

Please sign in to comment.