Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cluster indentifier #113

Merged
merged 10 commits into from
Jul 20, 2022
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ CONTROLLER_GEN_OLD ?= $(LOCALBINOLD)/controller-gen
ENVTEST ?= $(LOCALBIN)/setup-envtest

## Tool Versions
KUSTOMIZE_VERSION ?= v3.8.7
KUSTOMIZE_VERSION ?= v4.5.5
tomer-shefler marked this conversation as resolved.
Show resolved Hide resolved
CONTROLLER_TOOLS_VERSION ?= v0.9.0
CONTROLLER_TOOLS_OLD_VERSION ?= v0.6.2

Expand Down
3 changes: 2 additions & 1 deletion cbcontainers/communication/gateway/api_gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,15 @@ func (gateway *ApiGateway) getResourcePathWithAccountPath(resourceName string) s
return gateway.baseUrl(fmt.Sprintf("account/%s/%s", gateway.account, resourceName))
}

func (gateway *ApiGateway) RegisterCluster() error {
func (gateway *ApiGateway) RegisterCluster(clusterIdentifier string) error {
url := gateway.getResourcePathWithAccountPath("clusters")
resp, err := gateway.baseRequest().
SetBody(map[string]interface{}{
"name": gateway.cluster,
"components": gateway.agentComponents,
"labels": gateway.clusterLabels,
"inbounddefault": "allow",
"identifier": clusterIdentifier,
}).
Post(url)

Expand Down
9 changes: 6 additions & 3 deletions cbcontainers/processors/agent_processor.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
)

type APIGateway interface {
RegisterCluster() error
RegisterCluster(clusterIdentifier string) error
GetRegistrySecret() (*models.RegistrySecretValues, error)
GetCompatibilityMatrixEntryFor(operatorVersion string) (*models.OperatorCompatibility, error)
}
Expand All @@ -34,14 +34,17 @@ type AgentProcessor struct {
lastProcessedObject *cbcontainersv1.CBContainersAgent

log logr.Logger

clusterIdentifier string
}

func NewAgentProcessor(log logr.Logger, clusterRegistrarCreator APIGatewayCreator, operatorVersionProvider OperatorVersionProvider) *AgentProcessor {
func NewAgentProcessor(log logr.Logger, clusterRegistrarCreator APIGatewayCreator, operatorVersionProvider OperatorVersionProvider, clusterIdentifier string) *AgentProcessor {
return &AgentProcessor{
gatewayCreator: clusterRegistrarCreator,
lastProcessedObject: nil,
operatorVersionProvider: operatorVersionProvider,
log: log,
clusterIdentifier: clusterIdentifier,
}
}

Expand Down Expand Up @@ -88,7 +91,7 @@ func (processor *AgentProcessor) initializeIfNeeded(cbContainersCluster *cbconta
}

processor.log.Info("Calling register cluster")
if err := gateway.RegisterCluster(); err != nil {
if err := gateway.RegisterCluster(processor.clusterIdentifier); err != nil {
return err
}

Expand Down
12 changes: 7 additions & 5 deletions cbcontainers/processors/agent_processor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ var (
AccessToken = test_utils.RandomString()
)

const mockIdentifier string = "00000000-0000-0000-0000-000000000000"

func testClusterProcessor(t *testing.T, setupAndAssert SetupAndAssertClusterProcessorTest) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
Expand All @@ -37,7 +39,7 @@ func testClusterProcessor(t *testing.T, setupAndAssert SetupAndAssertClusterProc
operatorVersionProviderMock: mocks.NewMockOperatorVersionProvider(ctrl),
}

processor := processors.NewAgentProcessor(logrTesting.NewTestLogger(t), mocksObjects.gatewayCreatorMock, mocksObjects.operatorVersionProviderMock)
processor := processors.NewAgentProcessor(logrTesting.NewTestLogger(t), mocksObjects.gatewayCreatorMock, mocksObjects.operatorVersionProviderMock, mockIdentifier)
setupAndAssert(mocksObjects, processor)
}

Expand All @@ -46,7 +48,7 @@ func setupValidMocksCalls(testMocks *ClusterProcessorTestMocks, times int) {
testMocks.gatewayMock.EXPECT().GetRegistrySecret().DoAndReturn(func() (*models.RegistrySecretValues, error) {
return &models.RegistrySecretValues{Data: map[string][]byte{test_utils.RandomString(): {}}}, nil
}).Times(times)
testMocks.gatewayMock.EXPECT().RegisterCluster().Return(nil).Times(times)
testMocks.gatewayMock.EXPECT().RegisterCluster(mockIdentifier).Return(nil).Times(times)
// this will skip the compatibility check
// for all tests that do not explicitly test that
testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("", operator.ErrNotSemVer).AnyTimes()
Expand Down Expand Up @@ -99,7 +101,7 @@ func TestProcessorReturnsErrorWhenCanNotRegisterCluster(t *testing.T) {
clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}}
testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil)
testMocks.gatewayMock.EXPECT().GetRegistrySecret().Return(&models.RegistrySecretValues{}, nil)
testMocks.gatewayMock.EXPECT().RegisterCluster().Return(fmt.Errorf(""))
testMocks.gatewayMock.EXPECT().RegisterCluster(mockIdentifier).Return(fmt.Errorf(""))
_, err := processor.Process(clusterCR, AccessToken)
require.Error(t, err)
})
Expand All @@ -110,7 +112,7 @@ func TestProcessorReturnsErrorWhenOperatorVersionProviderReturnsUnknownError(t *
clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}}
testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil)
testMocks.gatewayMock.EXPECT().GetRegistrySecret().Return(&models.RegistrySecretValues{}, nil)
testMocks.gatewayMock.EXPECT().RegisterCluster().Return(nil)
testMocks.gatewayMock.EXPECT().RegisterCluster(mockIdentifier).Return(nil)
testMocks.operatorVersionProviderMock.EXPECT().GetOperatorVersion().Return("", fmt.Errorf("intentional unknown error"))
_, err := processor.Process(clusterCR, AccessToken)
require.Error(t, err)
Expand Down Expand Up @@ -171,7 +173,7 @@ func TestCheckCompatibilityCompatibleVersions(t *testing.T) {
clusterCR := &cbcontainersv1.CBContainersAgent{Spec: cbcontainersv1.CBContainersAgentSpec{Version: "1.0.0", Account: test_utils.RandomString(), ClusterName: test_utils.RandomString()}}
testMocks.gatewayCreatorMock.EXPECT().CreateGateway(gomock.Any(), gomock.Any()).Return(testMocks.gatewayMock, nil)
testMocks.gatewayMock.EXPECT().GetRegistrySecret().Return(&models.RegistrySecretValues{}, nil)
testMocks.gatewayMock.EXPECT().RegisterCluster().Return(nil)
testMocks.gatewayMock.EXPECT().RegisterCluster(mockIdentifier).Return(nil)
testCase.setup(testMocks)

values, err := processor.Process(clusterCR, AccessToken)
Expand Down
8 changes: 4 additions & 4 deletions cbcontainers/processors/mocks/mock_api_gateway.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,12 @@ rules:
- patch
- update
- watch
- apiGroups:
- ""
resources:
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
Expand Down
6 changes: 3 additions & 3 deletions controllers/cbcontainersagent_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,8 @@ type AgentProcessor interface {

type CBContainersAgentController struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme

Log logr.Logger
Scheme *runtime.Scheme
ClusterProcessor AgentProcessor
StateApplier StateApplier
K8sVersion string
Expand Down Expand Up @@ -80,6 +79,7 @@ func (r *CBContainersAgentController) getContainersAgentObject(ctx context.Conte
// +kubebuilder:rbac:groups={apps,core},resources={deployments,services,daemonsets},verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources={validatingwebhookconfigurations,mutatingwebhookconfigurations},verbs=*
// +kubebuilder:rbac:groups={core},resources={nodes},verbs=list
// +kubebuilder:rbac:groups={core},resources={namespaces},verbs=get
// +kubebuilder:rbac:groups={policy},resources={podsecuritypolicies},verbs=use,resourceNames={cbcontainers-manager-psp}

func (r *CBContainersAgentController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
Expand Down
34 changes: 30 additions & 4 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@ import (
"context"
"flag"
"fmt"
"os"

"github.com/vmware/cbcontainers-operator/cbcontainers/state"
"github.com/vmware/cbcontainers-operator/cbcontainers/state/agent_applyment"
"github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment"
"github.com/vmware/cbcontainers-operator/cbcontainers/state/operator"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"os"

coreV1 "k8s.io/api/core/v1"

Expand All @@ -39,20 +41,22 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"

"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"

operatorcontainerscarbonblackiov1 "github.com/vmware/cbcontainers-operator/api/v1"
certificatesUtils "github.com/vmware/cbcontainers-operator/cbcontainers/utils/certificates"
"github.com/vmware/cbcontainers-operator/controllers"
// +kubebuilder:scaffold:imports
tomer-shefler marked this conversation as resolved.
Show resolved Hide resolved
)

var (
scheme = runtime.NewScheme()
setupLog = ctrl.Log.WithName("setup")
)

const NamespaceIdentifier = "kube-system"

func init() {
utilruntime.Must(clientgoscheme.AddToScheme(scheme))

Expand Down Expand Up @@ -91,6 +95,28 @@ func main() {
os.Exit(1)
}

setupLog.Info(fmt.Sprintf("Getting Cluster Identifier: %v uid", NamespaceIdentifier))
config, err := rest.InClusterConfig()
if err != nil {
setupLog.Error(err, "unable to get the in cluster rest config")
tomer-shefler marked this conversation as resolved.
Show resolved Hide resolved
os.Exit(1)
}
// creates the clientset
client, err := kubernetes.NewForConfig(config)
if err != nil {
setupLog.Error(err, "unable to create the client")
os.Exit(1)
}

kubeSystem, err := client.CoreV1().Namespaces().Get(context.TODO(), NamespaceIdentifier, metav1.GetOptions{})
if err != nil {
setupLog.Error(err, fmt.Sprintf("unable to get the %v namespace", NamespaceIdentifier))
os.Exit(1)
}
clusterIdentifier := string(kubeSystem.UID)

setupLog.Info(fmt.Sprintf("Cluster Identifier: %v", clusterIdentifier))

setupLog.Info("Getting Nodes list")
nodesList := &coreV1.NodeList{}
if err := mgr.GetAPIReader().List(context.Background(), nodesList); err != nil || nodesList.Items == nil || len(nodesList.Items) < 1 {
Expand All @@ -106,7 +132,7 @@ func main() {
Log: cbContainersAgentLogger,
Scheme: mgr.GetScheme(),
K8sVersion: k8sVersion,
ClusterProcessor: processors.NewAgentProcessor(cbContainersAgentLogger, processors.NewDefaultGatewayCreator(), operator.NewEnvVersionProvider()),
ClusterProcessor: processors.NewAgentProcessor(cbContainersAgentLogger, processors.NewDefaultGatewayCreator(), operator.NewEnvVersionProvider(), clusterIdentifier),
StateApplier: state.NewStateApplier(agent_applyment.NewAgentComponent(applyment.NewComponentApplier(mgr.GetClient())), k8sVersion, certificatesUtils.NewCertificateCreator(), cbContainersAgentLogger),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "CBContainersAgent")
Expand Down